diff --git a/.github/workflows/atcoder.yaml b/.github/workflows/atcoder.yaml new file mode 100644 index 0000000..570e761 --- /dev/null +++ b/.github/workflows/atcoder.yaml @@ -0,0 +1,45 @@ +name: Test (atcoder) + +on: + push: + branches: ['**'] + tags-ignore: ['*'] + paths: ['atcoder/*'] + + workflow_dispatch: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + + steps: + - name: Set up Go 1.15 + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: Checkout Project + uses: actions/checkout@v2 + + - name: Cache Dependencies + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-go- + + - name: Run Tests + run: go test -v -coverprofile=c.out ./atcoder + env: + BROWSER_HEADLESS: + BROWSER_BINARY: google-chrome + + ATCODER_USERNAME: cptools + ATCODER_PASSWORD: ${{ secrets.ATCODER_PASSWORD }} + + - name: Upload Coverage + uses: shogo82148/actions-goveralls@v1 + with: + path-to-profile: c.out + diff --git a/.gitignore b/.gitignore index 839a56d..81a8163 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ # Dependency directories (remove the comment below to include it) vendor/ + +.env \ No newline at end of file diff --git a/atcoder/atcoder.go b/atcoder/atcoder.go new file mode 100644 index 0000000..7b931e3 --- /dev/null +++ b/atcoder/atcoder.go @@ -0,0 +1,88 @@ +package atcoder + +import ( + "fmt" + "regexp" + "strings" + + "github.com/cp-tools/cpt-lib/util" + + "github.com/go-rod/rod" +) + +type ( + // Args holds specifier details parsed by + // Parse() function. All methods use this + // at the core. + Args struct { + Contest string + Problem string + } + + page struct { + *rod.Page + } +) + +// Errors returned by library. +var ( + ErrInvalidSpecifier = fmt.Errorf("invalid specifier data") +) + +var ( + hostURL = "https://atcoder.jp" + + // Browser is the headless browser to use. + Browser *rod.Browser +) + +// Start initiates the automated browser to use. +func Start(headless bool, userDataDir, bin string) error { + bs, err := util.NewBrowser(headless, userDataDir, bin) + Browser = bs + + return err +} + +// Parse passed in specifier string to new Args struct. +// Validates parsed args and returns error if any. +func Parse(str string) (Args, error) { + var ( + rxCont = `(?P[A-Za-z0-9-]+)` + rxProb = `(?P[A-Za-z0-9_]+)` + + valRx = []string{ + `atcoder.jp\/contests\/` + rxCont + `$`, + `atcoder.jp\/contests\/` + rxCont + `\/tasks\/` + rxProb + `$`, + + `^` + rxCont + `$`, + `^` + rxCont + `\s+` + rxProb + `$`, + } + ) + + str = strings.TrimSpace(util.StrClean(str)) + if str == "" { + return Args{}, nil + } + + for _, rgx := range valRx { + re := regexp.MustCompile(rgx) + if re.MatchString(str) { + // https://stackoverflow.com/a/46202939/9606036 + match := re.FindStringSubmatch(str) + result := map[string]string{} + for i, name := range re.SubexpNames() { + if i != 0 && name != "" { + result[name] = match[i] + } + } + + arg := Args{ + Contest: result["cont"], + Problem: result["prob"], + } + return arg, nil + } + } + return Args{}, ErrInvalidSpecifier +} diff --git a/atcoder/atcoder_test.go b/atcoder/atcoder_test.go new file mode 100644 index 0000000..dd8a787 --- /dev/null +++ b/atcoder/atcoder_test.go @@ -0,0 +1,139 @@ +package atcoder + +import ( + "fmt" + "os" + "reflect" + "testing" + + "github.com/go-rod/rod" + "github.com/joho/godotenv" +) + +func login(usr, passwd string) (string, error) { + p, err := loadPage(fmt.Sprintf("%v/login", hostURL)) + if err != nil { + return "", err + } + defer p.Close() + + if _, err := p.Race().Element(`alert`).Handle(handleErrMsg). + Element(`footer.footer`).Do(); err != nil { + return "", err + } + + // Check if current user is logged in. + if handle := p.MustEval(`userScreenName`).String(); handle != "" { + return handle, nil + } + // Otherwise, login. + p.MustElement("#username").Input(usr) + p.MustElement("#password").Input(passwd) + p.MustElement("#submit").MustClick().WaitInvisible() + + if _, err := p.Race().ElementR(`.alert`, `Username or Password is incorrect`). + Handle(func(e *rod.Element) error { return fmt.Errorf(e.MustText()) }). + Element(`.navbar-right>li:last-child>a[class]`).Do(); err != nil { + return "", err + } + + handle := p.MustEval(`userScreenName`).String() + return handle, nil +} + +func getLoginCredentials() (string, string) { + // setup login access to use + usr := os.Getenv("ATCODER_USERNAME") + passwd := os.Getenv("ATCODER_PASSWORD") + return usr, passwd +} + +func TestMain(m *testing.M) { + // Load local .env file. + godotenv.Load() + + _, browserHeadless := os.LookupEnv("BROWSER_HEADLESS") + browserBin := os.Getenv("BROWSER_BINARY") + if err := Start(browserHeadless, "", browserBin); err != nil { + fmt.Println("Failed to start browser:", err) + os.Exit(1) + } + + if _, err := login(getLoginCredentials()); err != nil { + fmt.Println("Login failed:", err) + Browser.Close() + os.Exit(1) + } + + exitCode := m.Run() + + Browser.Close() + os.Exit(exitCode) +} + +func TestParse(t *testing.T) { + type args struct { + str string + } + tests := []struct { + name string + args args + want Args + wantErr bool + }{ + { + name: "Test #1", + args: args{"https://atcoder.jp/contests/acl1"}, + want: Args{"acl1", ""}, + wantErr: false, + }, + { + name: "Test #2", + args: args{"https://atcoder.jp/contests/m-solutions2020/tasks/m_solutions2020_a"}, + want: Args{"m-solutions2020", "m_solutions2020_a"}, + wantErr: false, + }, + { + name: "Test #3", + args: args{"arc107"}, + want: Args{"arc107", ""}, + wantErr: false, + }, + { + name: "Test #4", // Problem id need not match contest id. + args: args{"arc9999 aproblem"}, + want: Args{"arc9999", "aproblem"}, + wantErr: false, + }, + { + name: "Test #5", + args: args{"in_valid"}, + want: Args{}, + wantErr: true, + }, + { + name: "Test #6", + args: args{"in-valid in-valid"}, + want: Args{}, + wantErr: true, + }, + { + name: "Test #7", + args: args{""}, + want: Args{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Parse(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Parse() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/atcoder/contests.go b/atcoder/contests.go new file mode 100644 index 0000000..ff8c344 --- /dev/null +++ b/atcoder/contests.go @@ -0,0 +1,59 @@ +package atcoder + +import ( + "regexp" + "time" +) + +func (p *page) getCountdown() (time.Duration, error) { + // First check for virtual countdown. + if match := regexp.MustCompile(`var virtualStartTime = moment\("(.*)"\)`). + FindStringSubmatch(p.MustElement("html").MustHTML()); len(match) == 2 { + startTime, err := time.Parse(time.RFC3339Nano, match[1]) + if err != nil { + return 0, err + } + + dur := time.Until(startTime).Truncate(time.Second) + if dur < 0 { // Virtual already started; No countdown + dur = 0 + } + return dur, nil + } + + // Parse actual contest start time. + startTimeStr := p.MustEval("startTime._d.toISOString()").String() + startTime, err := time.Parse(time.RFC3339Nano, startTimeStr) + if err != nil { + return 0, err + } + + dur := time.Until(startTime).Truncate(time.Second) + if dur < 0 { // Contest already started; No countdown + dur = 0 + } + return dur, nil +} + +// GetCountdown ... +func (arg Args) GetCountdown() (time.Duration, error) { + link, err := arg.VirtualPage() + if err != nil { + return 0, err + } + + p, err := loadPage(link) + if err != nil { + return 0, err + } + defer p.Close() + + _, err = p.Race().Element(`.alert`).Handle(handleErrMsg). + Element(`footer.footer`).Do() + + if err != nil { + return 0, err + } + + return p.getCountdown() +} diff --git a/atcoder/contests_test.go b/atcoder/contests_test.go new file mode 100644 index 0000000..ec7266a --- /dev/null +++ b/atcoder/contests_test.go @@ -0,0 +1,40 @@ +package atcoder + +import ( + "testing" + "time" +) + +func TestArgs_GetCountdown(t *testing.T) { + tests := []struct { + name string + arg Args + want time.Duration + wantErr bool + }{ + { + name: "Test #1", + arg: Args{"abc180", ""}, + want: 0, + wantErr: false, + }, + { + name: "Test #2", + arg: Args{"InVaLiD123", ""}, + want: 0, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.arg.GetCountdown() + if (err != nil) != tt.wantErr { + t.Errorf("Args.GetCountdown() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Args.GetCountdown() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/atcoder/pages.go b/atcoder/pages.go new file mode 100644 index 0000000..4c9e5c4 --- /dev/null +++ b/atcoder/pages.go @@ -0,0 +1,24 @@ +package atcoder + +import "fmt" + +// DashboardPage returns link to dashboard of contest +func (arg Args) DashboardPage() (link string, err error) { + if arg.Contest == "" { + return "", ErrInvalidSpecifier + } + + link = fmt.Sprintf("%v/contests/%v", hostURL, arg.Contest) + return +} + +// VirtualPage returns link to virtual contest tab. +func (arg Args) VirtualPage() (link string, err error) { + if arg.Contest == "" { + return "", ErrInvalidSpecifier + } + + dashboardLink, _ := arg.DashboardPage() + link = fmt.Sprintf("%v/virtual", dashboardLink) + return +} diff --git a/atcoder/pages_test.go b/atcoder/pages_test.go new file mode 100644 index 0000000..af35abe --- /dev/null +++ b/atcoder/pages_test.go @@ -0,0 +1,71 @@ +package atcoder + +import "testing" + +func TestArgs_DashboardPage(t *testing.T) { + tests := []struct { + name string + arg Args + wantLink string + wantErr bool + }{ + { + name: "Test #1", + arg: Args{"hhkb2020", ""}, + wantLink: "https://atcoder.jp/contests/hhkb2020", + wantErr: false, + }, + { + name: "Test #2", + arg: Args{}, + wantLink: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotLink, err := tt.arg.DashboardPage() + if (err != nil) != tt.wantErr { + t.Errorf("Args.DashboardPage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotLink != tt.wantLink { + t.Errorf("Args.DashboardPage() = %v, want %v", gotLink, tt.wantLink) + } + }) + } +} + +func TestArgs_VirtualPage(t *testing.T) { + tests := []struct { + name string + arg Args + wantLink string + wantErr bool + }{ + { + name: "Test #1", + arg: Args{"tokiomarine2020", ""}, + wantLink: "https://atcoder.jp/contests/tokiomarine2020/virtual", + wantErr: false, + }, + { + name: "Test #2", + arg: Args{}, + wantLink: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotLink, err := tt.arg.VirtualPage() + if (err != nil) != tt.wantErr { + t.Errorf("Args.DashboardPage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotLink != tt.wantLink { + t.Errorf("Args.DashboardPage() = %v, want %v", gotLink, tt.wantLink) + } + }) + } +} diff --git a/atcoder/utils.go b/atcoder/utils.go new file mode 100644 index 0000000..53d8988 --- /dev/null +++ b/atcoder/utils.go @@ -0,0 +1,27 @@ +package atcoder + +import ( + "fmt" + + "github.com/cp-tools/cpt-lib/util" + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" +) + +func loadPage(link string) (*page, error) { + // Blocking these files results in faster page loading. + resourcesToBlock := []proto.NetworkResourceType{ + proto.NetworkResourceTypeFont, + proto.NetworkResourceTypeMedia, + proto.NetworkResourceTypeImage, + proto.NetworkResourceTypeStylesheet, + } + + p, err := util.NewPage(Browser, link, resourcesToBlock) + return &page{p}, err +} + +func handleErrMsg(e *rod.Element) error { + // There should be no notification. + return fmt.Errorf(e.MustText()) +} diff --git a/go.mod b/go.mod index fbd74d6..9ebd4d9 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,8 @@ module github.com/cp-tools/cpt-lib go 1.14 -require github.com/PuerkitoBio/goquery v1.5.1 +require ( + github.com/PuerkitoBio/goquery v1.6.0 + github.com/go-rod/rod v0.87.2 + github.com/joho/godotenv v1.3.0 +) diff --git a/go.sum b/go.sum index 6b8db65..b2c07e0 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,21 @@ -github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= +github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/go-rod/rod v0.87.2 h1:zlos09t7oV1kDxgz26oRvcYX0mQkjeJSNJugq9MUCno= +github.com/go-rod/rod v0.87.2/go.mod h1:UEYVhPXlrtQIL1Nlnq/fPY7Q5pB708UIEy5IKtNVuFg= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0= +github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs= +github.com/ysmood/got v0.8.9 h1:fUbi0c03DSNdo82DWd4m6l63WswtJDqwBPTcrrKMUHU= +github.com/ysmood/got v0.8.9/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/gotrace v0.1.1 h1:1H9L1Rc0o/80+2Vtm3vn85rNVpNOo94/wf+G5qW0JY8= +github.com/ysmood/gotrace v0.1.1/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU= +github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.6.11 h1:9y8/5v9FjHFXo81vZyh+VQTQCLs+aFjuJJqbpn33TLg= +github.com/ysmood/leakless v0.6.11/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= diff --git a/util/browser.go b/util/browser.go new file mode 100644 index 0000000..7bc1dec --- /dev/null +++ b/util/browser.go @@ -0,0 +1,88 @@ +package util + +import ( + "os" + "path/filepath" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/proto" +) + +// NewBrowser initiates the automated browser to use. +func NewBrowser(headless bool, userDataDir, bin string) (*rod.Browser, error) { + // Launch browser. + launchBrowser := func(controlURL string) (*rod.Browser, error) { + b := rod.New().ControlURL(controlURL) + if err := b.Connect(); err != nil { + return nil, err + } + return b, nil + } + + // Store data in cache (to reduce time). + cacheDir, _ := os.UserCacheDir() + cacheUserDataDir := filepath.Join(cacheDir, "cp-tools", "cpt-lib", bin) + + // Initiate the browser to use. + l := launcher.New(). + UserDataDir(cacheUserDataDir). + Headless(headless). + Bin(bin) + + controlURL, err := l.Launch() + if err != nil { + return nil, err + } + + Browser, err := launchBrowser(controlURL) + if err != nil { + return nil, err + } + + // Load temporary browser to extract cookies only if path exists. + if file, err := os.Stat(userDataDir); err == nil && file.IsDir() { + // Initiate browser to extract cookies from. + cookiesl := launcher.NewUserMode(). + UserDataDir(userDataDir). + Headless(true). + Bin(bin) + + cookiesControlURL, err := cookiesl.Launch() + if err != nil { + return nil, err + } + + cookiesBrowser, err := launchBrowser(cookiesControlURL) + if err != nil { + return nil, err + } + defer cookiesBrowser.Close() + // Copy cookies of user. + Browser.MustSetCookies(cookiesBrowser.MustGetCookies()) + } + + return Browser, nil +} + +// NewPage loads the given link in a new browser tab. +func NewPage(browser *rod.Browser, link string, block []proto.NetworkResourceType) (*rod.Page, error) { + page, err := browser.Page(proto.TargetCreateTarget{URL: link}) + if err != nil { + return nil, err + } + + router := page.HijackRequests() + router.MustAdd("*", func(h *rod.Hijack) { + for _, b := range block { + if h.Request.Type() == b { + h.Response.Fail(proto.NetworkErrorReasonBlockedByClient) + return + } + } + h.ContinueRequest(&proto.FetchContinueRequest{}) + }) + go router.Run() + + return page, nil +} diff --git a/util/strings.go b/util/strings.go new file mode 100644 index 0000000..9468eef --- /dev/null +++ b/util/strings.go @@ -0,0 +1,29 @@ +package util + +import ( + "math/rand" + "regexp" +) + +// StrClean replaces all unicode space +// characters in the string with ascii space. +func StrClean(str string) string { + re := regexp.MustCompile(`\p{Z}`) + return re.ReplaceAllString(str, " ") +} + +// StrRandom generates a random string of length n. +// The returned string is strictly alpha-numeric. +// Attrib: https://stackoverflow.com/a/31832326/9606036. +func StrRandom(n int) string { + const charBytes = "abcdefghijklmnopqrstuvwxyz" + + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + "0123456789" + + b := make([]byte, n) + for i := range b { + b[i] = charBytes[rand.Intn(len(charBytes))] + } + + return string(b) +}