preset scraper added
This commit is contained in:
parent
bfdfc634e4
commit
0ecf0ddec1
@ -4,7 +4,7 @@ tmp_dir = "tmp"
|
||||
|
||||
[build]
|
||||
args_bin = []
|
||||
bin = "tmp\\main.exe --config-file ./config/default --scope rutracker"
|
||||
bin = "tmp\\main.exe --config-file config/default --scope rutracker"
|
||||
cmd = "go build -o ./tmp/main.exe ./cmd"
|
||||
delay = 1000
|
||||
exclude_dir = ["assets", "tmp", "vendor", "testdata"]
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -31,4 +31,4 @@ fabric.properties
|
||||
.env
|
||||
tmp/
|
||||
vendor/
|
||||
config/*.yaml
|
||||
config/secret.yaml
|
||||
@ -3,7 +3,7 @@
|
||||
<option name="myName" value="Project Default" />
|
||||
<inspection_tool class="GoDfaErrorMayBeNotNil" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<functions>
|
||||
<function importPath="git.amok.space/yevhen/resource-scraper/pkg/repository" name="fetch" />
|
||||
<function importPath="net/http" name="Get" />
|
||||
</functions>
|
||||
</inspection_tool>
|
||||
</profile>
|
||||
|
||||
26
cmd/main.go
26
cmd/main.go
@ -1,16 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
scraper "git.amok.space/yevhen/resource-scraper/internal"
|
||||
"git.amok.space/yevhen/resource-scraper/internal/config"
|
||||
)
|
||||
|
||||
var st time.Time
|
||||
|
||||
//var tokenAuth *jwtauth.JWTAuth
|
||||
|
||||
func init() {
|
||||
st = time.Now()
|
||||
scraper.ParseFlags()
|
||||
config.New()
|
||||
|
||||
/*verifyKey := []byte(viper.GetString("jwt.sign_key"))
|
||||
tokenAuth = jwtauth.New("HS256", verifyKey, nil)
|
||||
|
||||
// For debugging/example purposes, we generate and print
|
||||
// a sample jwt token with claims `user_id:123` here:
|
||||
_, tokenString, _ := tokenAuth.Encode(map[string]interface{}{"user_id": 123})
|
||||
slog.Info("jwt", "token", tokenString)*/
|
||||
}
|
||||
|
||||
func main() {
|
||||
defer func() {
|
||||
fmt.Printf("\nExecution took %v", time.Since(st))
|
||||
}()
|
||||
|
||||
//a, b := runtime.GOOS
|
||||
//fmt.Printf("%v - - - \n", runtime.GOOS)
|
||||
|
||||
scraper.Bootstrap()
|
||||
//fmt.Printf("%v\n", viper.Get("23").([]interface{}))
|
||||
// Кінґстонська стіна
|
||||
}
|
||||
|
||||
3
config/default.yaml
Normal file
3
config/default.yaml
Normal file
@ -0,0 +1,3 @@
|
||||
scope:
|
||||
allow: [info, web, rutracker, prescene]
|
||||
default: web
|
||||
@ -1,4 +0,0 @@
|
||||
db:
|
||||
driver: mysql
|
||||
data_source_name: user:passwd@(localhost:3306)/dbname
|
||||
|
||||
1
config/scope/info.yaml
Normal file
1
config/scope/info.yaml
Normal file
@ -0,0 +1 @@
|
||||
role: console
|
||||
69
config/scope/prescene.yaml
Normal file
69
config/scope/prescene.yaml
Normal file
@ -0,0 +1,69 @@
|
||||
role: console
|
||||
endpoint: https://scnlog.me/music/flac/
|
||||
endpoint-next: page/%s/
|
||||
sleep-before-next-iteration: 5s
|
||||
levels-to-scrape: 5
|
||||
groups:
|
||||
tags:
|
||||
neformat:
|
||||
- SHGZ
|
||||
remove:
|
||||
- BIGLOVE
|
||||
- CT
|
||||
- ENViED
|
||||
- GARLICKNOTS
|
||||
- Ouzo
|
||||
- PTCx
|
||||
- SHGZ
|
||||
- XiVERO
|
||||
ignore:
|
||||
- "401"
|
||||
- 6DM
|
||||
- AFO
|
||||
- AOVF
|
||||
- AOVF_Classics
|
||||
- AUDiOFiLE
|
||||
- BABAS
|
||||
- BBD
|
||||
- BEATOCUL
|
||||
- CALiFLAC
|
||||
- CEBAD
|
||||
- D2H
|
||||
- DALiAS
|
||||
- dh
|
||||
- DWM
|
||||
- EMP
|
||||
- ENRiCH
|
||||
- ERP
|
||||
- ESGFLAC
|
||||
- FATHEAD
|
||||
- FiXIE
|
||||
- FLACME
|
||||
- FrB
|
||||
- HOUND
|
||||
- JRO
|
||||
- KINDA
|
||||
- MAHOU
|
||||
- MFDOS
|
||||
- MLS
|
||||
- Mrflac
|
||||
- MUNDANE
|
||||
- MyDad
|
||||
- OBZEN
|
||||
- OTT
|
||||
- PERFECT
|
||||
- PLAYLiST
|
||||
- PTC
|
||||
- PWT
|
||||
- RAGEFLAC
|
||||
- RAWBEATS
|
||||
- RECTiFY
|
||||
- ROSiN
|
||||
- STAX
|
||||
- THEVOiD
|
||||
- TiMES
|
||||
- TM
|
||||
- TVRf
|
||||
- VEXED
|
||||
- WAVED
|
||||
- YARD
|
||||
3
config/scope/web.yaml
Normal file
3
config/scope/web.yaml
Normal file
@ -0,0 +1,3 @@
|
||||
role: web
|
||||
http:
|
||||
port: 19576
|
||||
6
config/secret.yaml-sample
Normal file
6
config/secret.yaml-sample
Normal file
@ -0,0 +1,6 @@
|
||||
db:
|
||||
driver: mysql
|
||||
data_source_name: <user:pass@(localhost:3306)/bd_name>
|
||||
jwt:
|
||||
ttl: 6h12m24s
|
||||
sign_key: <your key>
|
||||
20
go.mod
20
go.mod
@ -5,30 +5,46 @@ go 1.23.1
|
||||
require (
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/go-chi/cors v1.2.1
|
||||
github.com/go-chi/jwtauth/v5 v5.3.1
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
|
||||
github.com/go-sql-driver/mysql v1.8.1
|
||||
github.com/golang-module/carbon/v2 v2.3.12
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.19.0
|
||||
golang.org/x/net v0.29.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
|
||||
github.com/lestrrat-go/httpcc v1.0.1 // indirect
|
||||
github.com/lestrrat-go/httprc v1.0.4 // indirect
|
||||
github.com/lestrrat-go/iter v1.0.2 // indirect
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.20 // indirect
|
||||
github.com/lestrrat-go/option v1.0.1 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/segmentio/asm v1.2.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.27.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
golang.org/x/sys v0.25.0 // indirect
|
||||
golang.org/x/text v0.18.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
75
go.sum
75
go.sum
@ -1,9 +1,13 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
@ -12,8 +16,18 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
|
||||
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
|
||||
github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||
github.com/go-chi/jwtauth/v5 v5.3.1 h1:1ePWrjVctvp1tyBq5b/2ER8Th/+RbYc7x4qNsc5rh5A=
|
||||
github.com/go-chi/jwtauth/v5 v5.3.1/go.mod h1:6Fl2RRmWXs3tJYE1IQGX81FsPoGqDwq9c15j52R5q80=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
|
||||
github.com/golang-module/carbon/v2 v2.3.12 h1:VC1DwN1kBwJkh5MjXmTFryjs5g4CWyoM8HAHffZPX/k=
|
||||
github.com/golang-module/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
@ -24,6 +38,18 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
|
||||
github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
|
||||
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
|
||||
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
|
||||
github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8=
|
||||
github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
|
||||
github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
|
||||
github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.20 h1:sAgXuWS/t8ykxS9Bi2Qtn5Qhpakw1wrcjxChudjolCc=
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.20/go.mod h1:UlCSmKqw+agm5BsOBfEAbTvKsEApaGNqHAEUTv5PJC4=
|
||||
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
|
||||
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
|
||||
@ -43,6 +69,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
|
||||
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
|
||||
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
|
||||
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
|
||||
@ -58,6 +86,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
@ -65,16 +94,54 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
|
||||
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
|
||||
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
|
||||
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
|
||||
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
63
helper/parser/html.go
Normal file
63
helper/parser/html.go
Normal file
@ -0,0 +1,63 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func setClient(url string) (*http.Response, error) {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
|
||||
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func HTMLSourceFromURL(url string) (*html.Node, error) {
|
||||
resp, err := setClient(url)
|
||||
defer func(Body io.ReadCloser) {
|
||||
fmt.Printf("%v\n", Body == nil)
|
||||
|
||||
err = Body.Close()
|
||||
if err != nil {
|
||||
slog.Error("closing response body", "err", err)
|
||||
}
|
||||
}(resp.Body)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, errors.New("http status code: " + strconv.Itoa(resp.StatusCode))
|
||||
}
|
||||
|
||||
doc, err := html.Parse(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
func HTMLSource(htmlSource string) (*html.Node, error) {
|
||||
doc, err := html.Parse(strings.NewReader(htmlSource))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
|
||||
@ -5,16 +5,15 @@ import (
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/internal/config"
|
||||
"git.amok.space/yevhen/resource-scraper/internal/db"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/handler"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/repository"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/service"
|
||||
"git.amok.space/yevhen/resource-scraper/types"
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func Bootstrap() {
|
||||
config.New()
|
||||
//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
|
||||
|
||||
dbase := db.New()
|
||||
repos := repository.New(dbase)
|
||||
@ -22,10 +21,12 @@ func Bootstrap() {
|
||||
handlers := handler.New(services)
|
||||
|
||||
switch viper.GetString("role") {
|
||||
case types.RoleConsole:
|
||||
case constant.RoleConsole:
|
||||
fmt.Printf("init console console: %s\n", handlers.InitConsole())
|
||||
break
|
||||
case constant.RoleWeb:
|
||||
fmt.Printf("who: %s\n", handlers.InitRoutes())
|
||||
///http.Run()
|
||||
break
|
||||
}
|
||||
|
||||
///http.Run()
|
||||
}
|
||||
|
||||
@ -3,32 +3,62 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func New() {
|
||||
configFilePath := viper.GetString("config-file")
|
||||
configDir := "./" + filepath.Dir(configFilePath)
|
||||
configFilePath := viper.GetString(constant.FlagConfigFile)
|
||||
configName := filepath.Base(configFilePath)
|
||||
|
||||
dir := filepath.Dir(configFilePath)
|
||||
if dir == configName && dir == "." {
|
||||
configName = "default"
|
||||
}
|
||||
|
||||
configDir, _ := filepath.Abs(dir)
|
||||
|
||||
viper.SetConfigName(filepath.Base(configFilePath))
|
||||
viper.AddConfigPath(configDir)
|
||||
|
||||
err := viper.ReadInConfig() // Find and read the config file
|
||||
if err != nil { // Handle errors reading the config file
|
||||
viper.SetConfigName(configName)
|
||||
err := viper.ReadInConfig()
|
||||
if err != nil { // Handle errors reading the config file
|
||||
panic(fmt.Errorf("fatal error config file: %w", err))
|
||||
}
|
||||
|
||||
viper.SetDefault("ConfigDir", configDir)
|
||||
scope := viper.GetString("scope")
|
||||
if scope != "" {
|
||||
viper.SetConfigName(scope)
|
||||
viper.AddConfigPath(configDir)
|
||||
err := viper.MergeInConfig()
|
||||
if err != nil {
|
||||
log.Fatalf("fatal error config file: %v", err)
|
||||
return
|
||||
viper.SetConfigName("secret")
|
||||
err = viper.MergeInConfig()
|
||||
if err != nil { // Handle errors reading the config file
|
||||
panic(fmt.Errorf("fatal error secret file: %w", err))
|
||||
}
|
||||
|
||||
slog.Info("using config", "path", filepath.Join(configDir, configName))
|
||||
viper.SetDefault(constant.CfgKeyConfigDir, configDir)
|
||||
|
||||
// Scopes validating
|
||||
scope := viper.GetString(constant.CfgKeyScopeEnable)
|
||||
scopesAllowed := viper.GetStringSlice("scope.allow")
|
||||
|
||||
if !slices.Contains(scopesAllowed, scope) {
|
||||
scope = viper.GetString("scope.default")
|
||||
|
||||
if scope == "" {
|
||||
scope = constant.ScopeInfo
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("scope", scope)
|
||||
|
||||
viper.SetConfigName(scope)
|
||||
viper.AddConfigPath(filepath.Join(configDir, "scope"))
|
||||
err = viper.MergeInConfig()
|
||||
if err == nil {
|
||||
viper.SetDefault(constant.CfgKeyScopeEnable, scope)
|
||||
} else {
|
||||
log.Fatalf("fatal error config file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,6 +8,8 @@ import (
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
const usage = `Music Database (MDB) server/cli craftware'
|
||||
@ -25,19 +27,18 @@ The Go runtime version: %s
|
||||
Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
|
||||
|
||||
const (
|
||||
maxPasswordLength = 20
|
||||
version = "0.1"
|
||||
defaultConfigPath = "./config/default"
|
||||
defaultConfigPath = "config/default"
|
||||
)
|
||||
|
||||
func ParseFlags() {
|
||||
flag.Bool("h", false, "")
|
||||
flag.Bool("help", false, "")
|
||||
flag.Bool("v", false, "")
|
||||
flag.Bool("version", false, "")
|
||||
flag.Bool("debug", false, "")
|
||||
flag.String("config-file", defaultConfigPath, "config file location used for the program")
|
||||
flag.String("scope", "", "")
|
||||
flag.Bool(constant.FlagHelpShort, false, "")
|
||||
flag.Bool(constant.FlagHelp, false, "")
|
||||
flag.Bool(constant.FlagVersionShort, false, "")
|
||||
flag.Bool(constant.FlagVersion, false, "")
|
||||
flag.Bool(constant.FlagDebug, false, "")
|
||||
flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location used for the program")
|
||||
flag.String(constant.FlagScopeEnable, "", "")
|
||||
|
||||
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
|
||||
pflag.Parse()
|
||||
@ -46,10 +47,10 @@ func ParseFlags() {
|
||||
log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
|
||||
}
|
||||
|
||||
if viper.GetBool("h") || viper.GetBool("help") {
|
||||
if viper.GetBool(constant.FlagHelpShort) || viper.GetBool(constant.FlagHelp) {
|
||||
msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
|
||||
fmt.Println(msg)
|
||||
} else if viper.GetBool("v") || viper.GetBool("version") {
|
||||
} else if viper.GetBool(constant.FlagVersionShort) || viper.GetBool(constant.FlagVersion) {
|
||||
fmt.Println("MDB version", version)
|
||||
}
|
||||
}
|
||||
|
||||
59
logpresece
Normal file
59
logpresece
Normal file
@ -0,0 +1,59 @@
|
||||
scope prescene
|
||||
FLOAT: 33.000000, 12.000000, 3
|
||||
false
|
||||
====================== 1 ==============================
|
||||
{Id:825647 Type:prescene TypeId:4313240 Title:Phaethon-Wielder_Of_The_Steel-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/phaethon-wielder_of_the_steel-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:09:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 2 ==============================
|
||||
{Id:825648 Type:prescene TypeId:4313238 Title:Helvetets_Port-Warlords-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/helvetets_port-warlords-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:08:33 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 4 ==============================
|
||||
{Id:825649 Type:prescene TypeId:4313236 Title:Circle_of_Ouroborus-Ajattoman_Uni-FI-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/circle_of_ouroborus-ajattoman_uni-fi-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:07:20 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 5 ==============================
|
||||
{Id:825650 Type:prescene TypeId:4313234 Title:Blitzkrieg-Blitzkrieg-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/blitzkrieg-blitzkrieg-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:06:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 6 ==============================
|
||||
{Id:825651 Type:prescene TypeId:4313233 Title:Antioch-Onward_With_Obliteration-EP-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/antioch-onward_with_obliteration-ep-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:05:27 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 8 ==============================
|
||||
{Id:825631 Type:prescene TypeId:4313221 Title:Wretchgod-Suffering_Upon_Suffering-EP-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-suffering_upon_suffering-ep-24bit-web-flac-2024-moonblood/ Created:2024-09-10 14:07:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 9 ==============================
|
||||
{Id:825632 Type:prescene TypeId:4313220 Title:Wretchgod-Stygian_Blood_Ritual-EP-16BIT-WEB-FLAC-2022-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-stygian_blood_ritual-ep-16bit-web-flac-2022-moonblood/ Created:2024-09-10 14:06:51 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 11 ==============================
|
||||
{Id:825633 Type:prescene TypeId:4313188 Title:1349-The_Infernal_Pathway-REPACK-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-repack-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:34:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 15 ==============================
|
||||
{Id:825634 Type:prescene TypeId:4313177 Title:1349-The_Infernal_Pathway-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:17:42 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 16 ==============================
|
||||
{Id:825635 Type:prescene TypeId:4313175 Title:1349-Revelations_of_the_Black_Flame-16BIT-WEB-FLAC-2009-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-revelations_of_the_black_flame-16bit-web-flac-2009-moonblood/ Created:2024-09-10 13:16:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 17 ==============================
|
||||
{Id:825636 Type:prescene TypeId:4313173 Title:1349-Massive_Cauldron_of_Chaos-16BIT-WEB-FLAC-2014-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-massive_cauldron_of_chaos-16bit-web-flac-2014-moonblood/ Created:2024-09-10 13:15:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 19 ==============================
|
||||
{Id:825637 Type:prescene TypeId:4313171 Title:1349-Liberation-16BIT-WEB-FLAC-2003-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-liberation-16bit-web-flac-2003-moonblood/ Created:2024-09-10 13:14:38 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 21 ==============================
|
||||
{Id:825638 Type:prescene TypeId:4313168 Title:1349-Hellfire-16BIT-WEB-FLAC-2005-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-hellfire-16bit-web-flac-2005-moonblood/ Created:2024-09-10 13:13:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 22 ==============================
|
||||
{Id:825639 Type:prescene TypeId:4313166 Title:1349-Demonoir-REISSUE-16BIT-WEB-FLAC-2011-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-demonoir-reissue-16bit-web-flac-2011-moonblood/ Created:2024-09-10 13:12:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 23 ==============================
|
||||
{Id:825640 Type:prescene TypeId:4313164 Title:1349-Beyond_the_Apocalypse-16BIT-WEB-FLAC-2004-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-beyond_the_apocalypse-16bit-web-flac-2004-moonblood/ Created:2024-09-10 13:12:00 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
Sleeping... 0
|
||||
false
|
||||
====================== 1 ==============================
|
||||
{Id:825652 Type:prescene TypeId:4300530 Title:Zuckerbaby--Zuckerbaby-CD-FLAC-1997-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/zuckerbaby-zuckerbaby-cd-flac-1997-meowzik/ Created:2024-08-31 18:17:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 2 ==============================
|
||||
{Id:825653 Type:prescene TypeId:4300529 Title:Silmarils--Vegas_76-FR-CD-FLAC-2000-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/silmarils-vegas_76-fr-cd-flac-2000-meowzik/ Created:2024-08-31 18:16:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 3 ==============================
|
||||
{Id:825654 Type:prescene TypeId:4300527 Title:Michael_Kaeshammer--Days_Like_These-CD-FLAC-2007-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/michael_kaeshammer-days_like_these-cd-flac-2007-meowzik/ Created:2024-08-31 18:14:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 4 ==============================
|
||||
{Id:825655 Type:prescene TypeId:4300521 Title:Butch_Walker--Letters-CD-FLAC-2004-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/butch_walker-letters-cd-flac-2004-meowzik/ Created:2024-08-31 18:09:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 16 ==============================
|
||||
{Id:825656 Type:prescene TypeId:4300320 Title:Embodiment_Elimination-Metamorphosis_Incarnate_Through_Genetic_Devastation-(ISR071-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/embodiment_elimination-metamorphosis_incarnate_through_genetic_devastation-isr071-24-cd-flac-2024-86d/ Created:2024-08-31 11:19:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 19 ==============================
|
||||
{Id:825657 Type:prescene TypeId:4300309 Title:Putrified_Degradation-Manufractured-(ISR072-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/putrified_degradation-manufractured-isr072-24-cd-flac-2024-86d/ Created:2024-08-31 10:45:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 20 ==============================
|
||||
{Id:825658 Type:prescene TypeId:4300307 Title:Post_Mortal_Possession-The_Dead_Space_Between_the_Stars-(LORD091)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/post_mortal_possession-the_dead_space_between_the_stars-lord091-cd-flac-2024-86d/ Created:2024-08-31 10:43:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 21 ==============================
|
||||
{Id:825659 Type:prescene TypeId:4300306 Title:Invirulant-Necrothuggin-(ISR070-24)-CDEP-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/invirulant-necrothuggin-isr070-24-cdep-flac-2024-86d/ Created:2024-08-31 10:41:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
====================== 22 ==============================
|
||||
{Id:825660 Type:prescene TypeId:4300304 Title:Dead_and_Dripping-Disillusioned_by_Excessive_Human_Consumption-(LORD092)-DEMO_REISSUE-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/dead_and_dripping-disillusioned_by_excessive_human_consumption-lord092-demo_reissue-cd-flac-2024-86d/ Created:2024-08-31 10:38:13 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
|
||||
Sleeping... 1
|
||||
scope: prescene
|
||||
[1 45]
|
||||
init console console: ps
|
||||
|
||||
Execution took 12.6276275s
|
||||
@ -1,7 +1,10 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/service"
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
@ -13,16 +16,17 @@ func New(services *service.Service) *Handler {
|
||||
}
|
||||
|
||||
func (h *Handler) InitConsole() string {
|
||||
return h.rutracker()
|
||||
switch viper.GetString(constant.CfgKeyScopeEnable) {
|
||||
case constant.ScopeRuTracker:
|
||||
return h.rutracker()
|
||||
case constant.ScopePrescene:
|
||||
return h.prescene()
|
||||
}
|
||||
|
||||
return "no scope chosen"
|
||||
}
|
||||
|
||||
//func (h *Handler) Base(services *service.Service) *Handler {
|
||||
// return &Handler{services: services}
|
||||
//}
|
||||
|
||||
//func (h *Handler) InitApi() *chi.Mux {
|
||||
// api := chi.NewRouter()
|
||||
// api.Get("/", web.ApiFallbackHandler)
|
||||
//
|
||||
// return api
|
||||
//}
|
||||
func (h *Handler) InitRoutes() string {
|
||||
//TODO:
|
||||
return "i am the web initiator"
|
||||
}
|
||||
|
||||
46
pkg/handler/prescene.go
Normal file
46
pkg/handler/prescene.go
Normal file
@ -0,0 +1,46 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) prescene() string {
|
||||
pagesToScrape := []string{"1"}
|
||||
levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
|
||||
if levels > 1 {
|
||||
var q = (60.0 / 100.0) * (100.0 / viper.GetFloat64(constant.CfgKeyLevelsToScrape))
|
||||
minute := float64(time.Now().Minute())
|
||||
if minute < 1 {
|
||||
minute = 1
|
||||
}
|
||||
|
||||
fmt.Printf("FLOAT: %f, %f, %v\n", minute, q, math.Ceil(minute/q))
|
||||
|
||||
hour := time.Now().Hour()
|
||||
if hour == 1 {
|
||||
hour = 25
|
||||
} else if hour == 0 {
|
||||
hour = 1
|
||||
}
|
||||
|
||||
page := hour * int(math.Ceil(minute/q))
|
||||
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
|
||||
}
|
||||
|
||||
_, err := h.services.Prescene.GetPage(pagesToScrape)
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||
}
|
||||
|
||||
fmt.Printf("%+v\n", pagesToScrape)
|
||||
|
||||
return "ps"
|
||||
}
|
||||
@ -12,10 +12,12 @@ func (h *Handler) rutracker() string {
|
||||
key := fmt.Sprintf("topic.%v", time.Now().Hour())
|
||||
topics := viper.GetStringSlice(key)
|
||||
|
||||
err := h.services.Rutracker.GetTopic(topics)
|
||||
_, err := h.services.Rutracker.GetTopic(topics)
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting topic: ", err.Error())
|
||||
slog.Error("error occurred while getting topic: ", "err", err.Error())
|
||||
}
|
||||
|
||||
//fmt.Printf("%+v", rt)
|
||||
|
||||
return "rt"
|
||||
}
|
||||
|
||||
108
pkg/repository/prescene.go
Normal file
108
pkg/repository/prescene.go
Normal file
@ -0,0 +1,108 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-shiori/dom"
|
||||
"github.com/golang-module/carbon/v2"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/helper/parser"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/repository/table"
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
type Prescene struct {
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
func NewPresceneRepository(db *sqlx.DB) *Prescene {
|
||||
return &Prescene{db: db}
|
||||
}
|
||||
|
||||
func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
||||
entries := make([]model.ExternalSources, 0)
|
||||
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
||||
//scope := viper.GetString(constant.CfgKeyScopeEnable)
|
||||
tags := viper.GetStringMapStringSlice("groups.tags")
|
||||
|
||||
for _, t := range pageNumbers {
|
||||
if t != "1" {
|
||||
endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
|
||||
}
|
||||
doc, err := parser.HTMLSourceFromURL(endpoint)
|
||||
//doc, err := parser.HTMLSourceFromURL("https://mdb.amok.space/$/scnlog.html")
|
||||
|
||||
if err != nil {
|
||||
slog.Error("Parse error", "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if doc == nil {
|
||||
slog.Warn("Document is nil", "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
|
||||
var es model.ExternalSources
|
||||
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
|
||||
|
||||
title := dom.QuerySelector(item, ".title")
|
||||
if title != nil {
|
||||
anchor := dom.QuerySelector(title, "h1 > a")
|
||||
if anchor != nil {
|
||||
es.Type = constant.ScopePrescene
|
||||
es.Title = dom.GetAttribute(anchor, "title")
|
||||
es.ExSource = dom.GetAttribute(anchor, "href")
|
||||
pattern := regexp.MustCompile(`(?is)-(\w+)$`)
|
||||
es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
|
||||
|
||||
for flag, groups := range tags {
|
||||
if slices.Contains(groups, es.Releaser) {
|
||||
es.A = flag
|
||||
es.H = flag
|
||||
columns = append(columns, "a", "h")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if es.A == constant.TagIgnore {
|
||||
slog.Info("Skipped", "releaser", es.Releaser)
|
||||
continue
|
||||
}
|
||||
|
||||
localtime := dom.QuerySelector(title, "small > span.localtime")
|
||||
if localtime != nil {
|
||||
lc := dom.GetAttribute(localtime, "data-lttime")
|
||||
es.Created = carbon.Parse(lc)
|
||||
}
|
||||
}
|
||||
|
||||
cls := dom.GetAttribute(item, "class")
|
||||
pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
|
||||
es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
|
||||
|
||||
//fmt.Println("====================== ", i, " ==============================")
|
||||
esModel := table.ExternalSources{Columns: columns}
|
||||
entry := esModel.InsertOnDuplicate(es, s.db)
|
||||
entries = append(entries, entry)
|
||||
|
||||
//fmt.Printf("%+v\n", entry)
|
||||
}
|
||||
|
||||
//fmt.Println("Sleeping...", j)
|
||||
time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
|
||||
}
|
||||
|
||||
//fmt.Printf("scope: %v\n", scope)
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
@ -2,22 +2,18 @@ package repository
|
||||
|
||||
import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types"
|
||||
"git.amok.space/yevhen/resource-scraper/types/interface"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
types.Rutracker
|
||||
_interface.Rutracker
|
||||
_interface.Prescene
|
||||
}
|
||||
|
||||
func New(db *sqlx.DB) *Repository {
|
||||
switch viper.GetString("scope") {
|
||||
case types.RuTracker:
|
||||
return &Repository{
|
||||
Rutracker: NewRutracker(db),
|
||||
}
|
||||
return &Repository{
|
||||
Rutracker: NewRutrackerRepository(db),
|
||||
Prescene: NewPresceneRepository(db),
|
||||
}
|
||||
|
||||
return &Repository{}
|
||||
}
|
||||
|
||||
@ -8,60 +8,59 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/golang-module/carbon/v2"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
iface "git.amok.space/yevhen/resource-scraper/types"
|
||||
"git.amok.space/yevhen/resource-scraper/types/table"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/repository/table"
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
"git.amok.space/yevhen/resource-scraper/types/resource"
|
||||
)
|
||||
|
||||
type Rutracker struct {
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
func NewRutracker(db *sqlx.DB) *Rutracker {
|
||||
func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
|
||||
return &Rutracker{db: db}
|
||||
}
|
||||
|
||||
func (s *Rutracker) GetTopic(topics []string) error {
|
||||
endpoint := viper.GetString("endpoint")
|
||||
func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
||||
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
||||
entries := make([]model.ExternalSources, 0)
|
||||
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
|
||||
|
||||
for _, t := range topics {
|
||||
topic, err := fetch(fmt.Sprintf(endpoint, t))
|
||||
if err != nil {
|
||||
slog.Error("couldn't parse topic data", "err", err.Error())
|
||||
return entries, err
|
||||
}
|
||||
|
||||
for i, e := range topic.Entry {
|
||||
var id int
|
||||
var es table.ExternalSources
|
||||
|
||||
for _, e := range topic.Entry {
|
||||
var es model.ExternalSources
|
||||
u, _ := url.Parse(e.Link.Href)
|
||||
es.Type = "rutracker"
|
||||
|
||||
es.Type = constant.ScopeRuTracker
|
||||
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
|
||||
es.Title = e.Title
|
||||
es.TypeSubsectionId, _ = strconv.Atoi(t)
|
||||
es.Releaser = e.Author.Name
|
||||
es.Created, _ = time.Parse(time.RFC3339, e.Updated)
|
||||
created := es.Created.Format(iface.DateTimeFormat)
|
||||
es.Created = carbon.Parse(e.Updated)
|
||||
|
||||
query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable)
|
||||
|
||||
row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created)
|
||||
if err = row.Scan(&id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>")
|
||||
esModel := table.ExternalSources{Columns: columns}
|
||||
entry := esModel.InsertOnDuplicate(es, s.db)
|
||||
entries = append(entries, entry)
|
||||
//fmt.Printf("%+v\n\n\n", entry)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
|
||||
func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
|
||||
resp, err := http.Get(endpoint)
|
||||
if err != nil {
|
||||
slog.Error("couldn't fetch data", endpoint, err.Error())
|
||||
@ -75,7 +74,7 @@ func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
|
||||
}
|
||||
}(resp.Body)
|
||||
|
||||
topic := &iface.RutrackerAtomTopic{}
|
||||
topic := &resource.RutrackerAtomTopic{}
|
||||
|
||||
if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
|
||||
return nil, err
|
||||
|
||||
34
pkg/repository/table/external_sources.go
Normal file
34
pkg/repository/table/external_sources.go
Normal file
@ -0,0 +1,34 @@
|
||||
package table
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
type ExternalSources struct {
|
||||
Columns []string
|
||||
}
|
||||
|
||||
func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.DB) model.ExternalSources {
|
||||
stmt := "INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE title=:title, created=:created RETURNING id"
|
||||
placeholders := strings.Join(f.Columns, ", :")
|
||||
placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
|
||||
query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
|
||||
|
||||
//fmt.Printf("%s\n", query)
|
||||
|
||||
if rows, err := db.NamedQuery(query, &es); err == nil {
|
||||
for rows.Next() {
|
||||
es.Error = rows.StructScan(&es)
|
||||
}
|
||||
} else {
|
||||
es.Error = err
|
||||
}
|
||||
|
||||
return es
|
||||
}
|
||||
18
pkg/service/prescene.go
Normal file
18
pkg/service/prescene.go
Normal file
@ -0,0 +1,18 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"git.amok.space/yevhen/resource-scraper/types/interface"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
type PresceneService struct {
|
||||
repo _interface.Prescene
|
||||
}
|
||||
|
||||
func NewPresceneService(repo _interface.Prescene) *PresceneService {
|
||||
return &PresceneService{repo: repo}
|
||||
}
|
||||
|
||||
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
||||
return p.repo.GetPage(pageNumbers)
|
||||
}
|
||||
@ -1,17 +1,18 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
iface "git.amok.space/yevhen/resource-scraper/types"
|
||||
"git.amok.space/yevhen/resource-scraper/types/interface"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
type RutrackerService struct {
|
||||
repo iface.Rutracker
|
||||
repo _interface.Rutracker
|
||||
}
|
||||
|
||||
func NewRutrackerService(repo iface.Rutracker) *RutrackerService {
|
||||
func NewRutrackerService(repo _interface.Rutracker) *RutrackerService {
|
||||
return &RutrackerService{repo: repo}
|
||||
}
|
||||
|
||||
func (s *RutrackerService) GetTopic(topic []string) error {
|
||||
func (s *RutrackerService) GetTopic(topic []string) ([]model.ExternalSources, error) {
|
||||
return s.repo.GetTopic(topic)
|
||||
}
|
||||
|
||||
@ -1,24 +1,18 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/repository"
|
||||
"git.amok.space/yevhen/resource-scraper/types"
|
||||
"git.amok.space/yevhen/resource-scraper/types/interface"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
types.Rutracker
|
||||
_interface.Rutracker
|
||||
_interface.Prescene
|
||||
}
|
||||
|
||||
func New(repos *repository.Repository) *Service {
|
||||
|
||||
switch viper.GetString("scope") {
|
||||
case types.RuTracker:
|
||||
return &Service{
|
||||
Rutracker: NewRutrackerService(repos.Rutracker),
|
||||
}
|
||||
return &Service{
|
||||
Rutracker: NewRutrackerService(repos.Rutracker),
|
||||
Prescene: NewPresceneService(repos.Prescene),
|
||||
}
|
||||
|
||||
return &Service{}
|
||||
}
|
||||
|
||||
10
types/constant/config.go
Normal file
10
types/constant/config.go
Normal file
@ -0,0 +1,10 @@
|
||||
package constant
|
||||
|
||||
const (
|
||||
CfgKeyScopeEnable = FlagScopeEnable
|
||||
CfgKeyConfigDir = "config-dir"
|
||||
CfgKeySleepBeforeNextIteration = "sleep-before-next-iteration"
|
||||
CfgKeyEndpoint = "endpoint"
|
||||
CfgKeyEndpointNext = "endpoint-next"
|
||||
CfgKeyLevelsToScrape = "levels-to-scrape"
|
||||
)
|
||||
18
types/constant/constants.go
Normal file
18
types/constant/constants.go
Normal file
@ -0,0 +1,18 @@
|
||||
package constant
|
||||
|
||||
const (
|
||||
ScopeRuTracker string = "rutracker"
|
||||
ScopePrescene string = "prescene"
|
||||
ScopeWeb string = "web"
|
||||
ScopeInfo string = "info"
|
||||
)
|
||||
|
||||
const (
|
||||
RoleConsole string = "console"
|
||||
RoleWeb string = "web"
|
||||
)
|
||||
|
||||
const (
|
||||
TagIgnore string = "ignore"
|
||||
ExternalSourcesTable string = "external_sources"
|
||||
)
|
||||
11
types/constant/flag.go
Normal file
11
types/constant/flag.go
Normal file
@ -0,0 +1,11 @@
|
||||
package constant
|
||||
|
||||
const (
|
||||
FlagHelp = "help"
|
||||
FlagHelpShort = "h"
|
||||
FlagDebug = "debug"
|
||||
FlagVersion = "version"
|
||||
FlagVersionShort = "v"
|
||||
FlagConfigFile = "config-file"
|
||||
FlagScopeEnable = "scope-enable"
|
||||
)
|
||||
@ -1,14 +0,0 @@
|
||||
package types
|
||||
|
||||
const (
|
||||
RuTracker string = "rutracker"
|
||||
)
|
||||
|
||||
const (
|
||||
RoleConsole string = "console"
|
||||
)
|
||||
|
||||
const (
|
||||
DateTimeFormat string = "2006-01-02 15:04:05"
|
||||
ExternalSourcesTable string = "external_sources"
|
||||
)
|
||||
11
types/interface/interfaces.go
Normal file
11
types/interface/interfaces.go
Normal file
@ -0,0 +1,11 @@
|
||||
package _interface
|
||||
|
||||
import "git.amok.space/yevhen/resource-scraper/types/model"
|
||||
|
||||
type Rutracker interface {
|
||||
GetTopic(topics []string) ([]model.ExternalSources, error)
|
||||
}
|
||||
|
||||
type Prescene interface {
|
||||
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
package types
|
||||
|
||||
type Rutracker interface {
|
||||
GetTopic(topics []string) error
|
||||
}
|
||||
42
types/model/external_sources.go
Normal file
42
types/model/external_sources.go
Normal file
@ -0,0 +1,42 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"github.com/golang-module/carbon/v2"
|
||||
)
|
||||
|
||||
type Type string
|
||||
|
||||
/*const (
|
||||
mmt Type = iota
|
||||
prescene
|
||||
rutracker
|
||||
locate
|
||||
darkabyss
|
||||
bfm
|
||||
trash
|
||||
nnmc
|
||||
stb
|
||||
bandcamp
|
||||
deathgrind
|
||||
)*/
|
||||
|
||||
type ExternalSources struct {
|
||||
Id int `json:"id" db:"id"`
|
||||
Type string `json:"type" db:"type"`
|
||||
TypeId int `json:"type_id" db:"type_id"`
|
||||
Title string `json:"title" db:"title"`
|
||||
TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
|
||||
Releaser string `json:"releaser" db:"releaser"`
|
||||
ExSource string `json:"ex_source" db:"eXsource"`
|
||||
Created carbon.Carbon `json:"created_at" db:"created"`
|
||||
Modified carbon.Carbon `json:"modified_at" db:"modified"`
|
||||
Contents string `json:"contents" db:"contents"`
|
||||
A string `json:"a" db:"a"`
|
||||
H string `json:"h" db:"h"`
|
||||
Fingerprint string `json:"fingerprint" db:"fingerprint"`
|
||||
FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
|
||||
Vid int `json:"vid" db:"vid"`
|
||||
G string `json:"g" db:"g"`
|
||||
ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
|
||||
Error error
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package types
|
||||
package resource
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
46
types/resource/scnlog.go
Normal file
46
types/resource/scnlog.go
Normal file
@ -0,0 +1,46 @@
|
||||
package resource
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
type ScnLogRss struct {
|
||||
XMLName xml.Name `xml:"rss"`
|
||||
Text string `xml:",chardata"`
|
||||
Version string `xml:"version,attr"`
|
||||
Content string `xml:"content,attr"`
|
||||
Wfw string `xml:"wfw,attr"`
|
||||
Dc string `xml:"dc,attr"`
|
||||
Atom string `xml:"atom,attr"`
|
||||
Sy string `xml:"sy,attr"`
|
||||
Slash string `xml:"slash,attr"`
|
||||
Channel struct {
|
||||
Text string `xml:",chardata"`
|
||||
Title string `xml:"title"`
|
||||
Link struct {
|
||||
Text string `xml:",chardata"`
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
Type string `xml:"type,attr"`
|
||||
} `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
LastBuildDate string `xml:"lastBuildDate"`
|
||||
Language string `xml:"language"`
|
||||
UpdatePeriod string `xml:"updatePeriod"`
|
||||
UpdateFrequency string `xml:"updateFrequency"`
|
||||
Generator string `xml:"generator"`
|
||||
Item []struct {
|
||||
Text string `xml:",chardata"`
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Comments string `xml:"comments"`
|
||||
Creator string `xml:"creator"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Category []string `xml:"category"`
|
||||
Guid struct {
|
||||
Text string `xml:",chardata"`
|
||||
IsPermaLink string `xml:"isPermaLink,attr"`
|
||||
} `xml:"guid"`
|
||||
Description string `xml:"description"`
|
||||
CommentRss string `xml:"commentRss"`
|
||||
} `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
@ -1,39 +0,0 @@
|
||||
package table
|
||||
|
||||
import "time"
|
||||
|
||||
type Type string
|
||||
|
||||
/*const (
|
||||
mmt Type = iota
|
||||
prescene
|
||||
rutracker
|
||||
locate
|
||||
darkabyss
|
||||
bfm
|
||||
trash
|
||||
nnmc
|
||||
stb
|
||||
bandcamp
|
||||
deathgrind
|
||||
)*/
|
||||
|
||||
type ExternalSources struct {
|
||||
Id int `json:"id" db:"id"`
|
||||
Type string `json:"type" db:"type"`
|
||||
TypeId int `json:"type_id" db:"type_id"`
|
||||
Title string `json:"title" db:"title"`
|
||||
TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
|
||||
Releaser string `json:"releaser" db:"releaser"`
|
||||
ExSource string `json:"ex_source" db:"eXsource"`
|
||||
Created time.Time `json:"created_at" db:"created"`
|
||||
Modified time.Time `json:"modified_at" db:"modified"`
|
||||
Contents string `json:"contents" db:"contents"`
|
||||
A string `json:"a" db:"a"`
|
||||
H string `json:"h" db:"h"`
|
||||
Fingerprint string `json:"fingerprint" db:"fingerprint"`
|
||||
FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
|
||||
Vid int `json:"vid" db:"vid"`
|
||||
G string `json:"g" db:"g"`
|
||||
ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user