yevhen/resource-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

preset scraper added

Browse Source
This commit is contained in:
Yevhen Odynets 2024-09-10 15:57:48 +03:00
parent bfdfc634e4
commit 0ecf0ddec1
38 changed files with 790 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.air.toml
View File

@ -4,7 +4,7 @@ tmp_dir = "tmp"
[build] [build]
args_bin = [] args_bin = []
bin = "tmp\\main.exe --config-file ./config/default --scope rutracker" bin = "tmp\\main.exe --config-file config/default --scope rutracker"
cmd = "go build -o ./tmp/main.exe ./cmd" cmd = "go build -o ./tmp/main.exe ./cmd"
delay = 1000 delay = 1000
exclude_dir = ["assets", "tmp", "vendor", "testdata"] exclude_dir = ["assets", "tmp", "vendor", "testdata"]

2
.gitignore vendored
View File

@ -31,4 +31,4 @@ fabric.properties
.env .env
tmp/ tmp/
vendor/ vendor/
config/*.yaml config/secret.yaml

2
.idea/inspectionProfiles/Project_Default.xml
View File

@ -3,7 +3,7 @@
<option name="myName" value="Project Default" /> <option name="myName" value="Project Default" />
<inspection_tool class="GoDfaErrorMayBeNotNil" enabled="true" level="WARNING" enabled_by_default="true"> <inspection_tool class="GoDfaErrorMayBeNotNil" enabled="true" level="WARNING" enabled_by_default="true">
<functions> <functions>
<function importPath="git.amok.space/yevhen/resource-scraper/pkg/repository" name="fetch" /> <function importPath="net/http" name="Get" />
</functions> </functions>
</inspection_tool> </inspection_tool>
</profile> </profile>

26
cmd/main.go
View File

@ -1,16 +1,42 @@
package main package main
import ( import (
"fmt"
"time"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
scraper "git.amok.space/yevhen/resource-scraper/internal" scraper "git.amok.space/yevhen/resource-scraper/internal"
"git.amok.space/yevhen/resource-scraper/internal/config"
) )
var st time.Time
//var tokenAuth *jwtauth.JWTAuth
func init() { func init() {
st = time.Now()
scraper.ParseFlags() scraper.ParseFlags()
config.New()
/*verifyKey := []byte(viper.GetString("jwt.sign_key"))
tokenAuth = jwtauth.New("HS256", verifyKey, nil)
// For debugging/example purposes, we generate and print
// a sample jwt token with claims `user_id:123` here:
_, tokenString, _ := tokenAuth.Encode(map[string]interface{}{"user_id": 123})
slog.Info("jwt", "token", tokenString)*/
} }
func main() { func main() {
defer func() {
fmt.Printf("\nExecution took %v", time.Since(st))
}()
//a, b := runtime.GOOS
//fmt.Printf("%v - - - \n", runtime.GOOS)
scraper.Bootstrap() scraper.Bootstrap()
//fmt.Printf("%v\n", viper.Get("23").([]interface{})) //fmt.Printf("%v\n", viper.Get("23").([]interface{}))
// Кінґстонська стіна
} }

3
config/default.yaml Normal file
View File

@ -0,0 +1,3 @@
scope:
allow: [info, web, rutracker, prescene]
default: web

4
config/default.yml-sample
View File

@ -1,4 +0,0 @@
db:
driver: mysql
data_source_name: user:passwd@(localhost:3306)/dbname

1
config/scope/info.yaml Normal file
View File

@ -0,0 +1 @@
role: console

69
config/scope/prescene.yaml Normal file
View File

@ -0,0 +1,69 @@
role: console
endpoint: https://scnlog.me/music/flac/
endpoint-next: page/%s/
sleep-before-next-iteration: 5s
levels-to-scrape: 5
groups:
tags:
neformat:
- SHGZ
remove:
- BIGLOVE
- CT
- ENViED
- GARLICKNOTS
- Ouzo
- PTCx
- SHGZ
- XiVERO
ignore:
- "401"
- 6DM
- AFO
- AOVF
- AOVF_Classics
- AUDiOFiLE
- BABAS
- BBD
- BEATOCUL
- CALiFLAC
- CEBAD
- D2H
- DALiAS
- dh
- DWM
- EMP
- ENRiCH
- ERP
- ESGFLAC
- FATHEAD
- FiXIE
- FLACME
- FrB
- HOUND
- JRO
- KINDA
- MAHOU
- MFDOS
- MLS
- Mrflac
- MUNDANE
- MyDad
- OBZEN
- OTT
- PERFECT
- PLAYLiST
- PTC
- PWT
- RAGEFLAC
- RAWBEATS
- RECTiFY
- ROSiN
- STAX
- THEVOiD
- TiMES
- TM
- TVRf
- VEXED
- WAVED
- YARD

0
config/rutracker.yml-sample → config/scope/rutracker.yaml
View File

3
config/scope/web.yaml Normal file
View File

@ -0,0 +1,3 @@
role: web
http:
port: 19576

6
config/secret.yaml-sample Normal file
View File

@ -0,0 +1,6 @@
db:
driver: mysql
data_source_name: <user:pass@(localhost:3306)/bd_name>
jwt:
ttl: 6h12m24s
sign_key: <your key>

20
go.mod
View File

@ -5,30 +5,46 @@ go 1.23.1
require ( require (
github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/chi/v5 v5.1.0
github.com/go-chi/cors v1.2.1 github.com/go-chi/cors v1.2.1
github.com/go-chi/jwtauth/v5 v5.3.1
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
github.com/go-sql-driver/mysql v1.8.1 github.com/go-sql-driver/mysql v1.8.1
github.com/golang-module/carbon/v2 v2.3.12
github.com/jmoiron/sqlx v1.4.0 github.com/jmoiron/sqlx v1.4.0
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.19.0 github.com/spf13/viper v1.19.0
golang.org/x/net v0.29.0
) )
require ( require (
filippo.io/edwards25519 v1.1.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/httprc v1.0.4 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/jwx/v2 v2.0.20 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/magiconair/properties v1.8.7 // indirect github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cast v1.6.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/atomic v1.9.0 // indirect go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/sys v0.18.0 // indirect golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.14.0 // indirect golang.org/x/text v0.18.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
) )

75
go.sum
View File

@ -1,9 +1,13 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@ -12,8 +16,18 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-chi/jwtauth/v5 v5.3.1 h1:1ePWrjVctvp1tyBq5b/2ER8Th/+RbYc7x4qNsc5rh5A=
github.com/go-chi/jwtauth/v5 v5.3.1/go.mod h1:6Fl2RRmWXs3tJYE1IQGX81FsPoGqDwq9c15j52R5q80=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
github.com/golang-module/carbon/v2 v2.3.12 h1:VC1DwN1kBwJkh5MjXmTFryjs5g4CWyoM8HAHffZPX/k=
github.com/golang-module/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@ -24,6 +38,18 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8=
github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
github.com/lestrrat-go/jwx/v2 v2.0.20 h1:sAgXuWS/t8ykxS9Bi2Qtn5Qhpakw1wrcjxChudjolCc=
github.com/lestrrat-go/jwx/v2 v2.0.20/go.mod h1:UlCSmKqw+agm5BsOBfEAbTvKsEApaGNqHAEUTv5PJC4=
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
@ -43,6 +69,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
@ -58,6 +86,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
@ -65,16 +94,54 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

63
helper/parser/html.go Normal file
View File

@ -0,0 +1,63 @@
package parser
import (
"errors"
"fmt"
"io"
"log"
"log/slog"
"net/http"
"strconv"
"strings"
"golang.org/x/net/html"
)
func setClient(url string) (*http.Response, error) {
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalln(err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
return client.Do(req)
}
func HTMLSourceFromURL(url string) (*html.Node, error) {
resp, err := setClient(url)
defer func(Body io.ReadCloser) {
fmt.Printf("%v\n", Body == nil)
err = Body.Close()
if err != nil {
slog.Error("closing response body", "err", err)
}
}(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode >= 400 {
return nil, errors.New("http status code: " + strconv.Itoa(resp.StatusCode))
}
doc, err := html.Parse(resp.Body)
if err != nil {
return nil, err
}
return doc, nil
}
func HTMLSource(htmlSource string) (*html.Node, error) {
doc, err := html.Parse(strings.NewReader(htmlSource))
if err != nil {
return nil, err
}
return doc, nil
}
//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang

13
internal/bootstrap.go
View File

@ -5,16 +5,15 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/internal/config"
"git.amok.space/yevhen/resource-scraper/internal/db" "git.amok.space/yevhen/resource-scraper/internal/db"
"git.amok.space/yevhen/resource-scraper/pkg/handler" "git.amok.space/yevhen/resource-scraper/pkg/handler"
"git.amok.space/yevhen/resource-scraper/pkg/repository" "git.amok.space/yevhen/resource-scraper/pkg/repository"
"git.amok.space/yevhen/resource-scraper/pkg/service" "git.amok.space/yevhen/resource-scraper/pkg/service"
"git.amok.space/yevhen/resource-scraper/types" "git.amok.space/yevhen/resource-scraper/types/constant"
) )
func Bootstrap() { func Bootstrap() {
config.New() //https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
dbase := db.New() dbase := db.New()
repos := repository.New(dbase) repos := repository.New(dbase)
@ -22,10 +21,12 @@ func Bootstrap() {
handlers := handler.New(services) handlers := handler.New(services)
switch viper.GetString("role") { switch viper.GetString("role") {
case types.RoleConsole: case constant.RoleConsole:
fmt.Printf("init console console: %s\n", handlers.InitConsole()) fmt.Printf("init console console: %s\n", handlers.InitConsole())
break break
case constant.RoleWeb:
fmt.Printf("who: %s\n", handlers.InitRoutes())
///http.Run()
break
} }
///http.Run()
} }

60
internal/config/config.go
View File

@ -3,32 +3,62 @@ package config
import ( import (
"fmt" "fmt"
"log" "log"
"log/slog"
"path/filepath" "path/filepath"
"slices"
"github.com/spf13/viper" "github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
) )
func New() { func New() {
configFilePath := viper.GetString("config-file") configFilePath := viper.GetString(constant.FlagConfigFile)
configDir := "./" + filepath.Dir(configFilePath) configName := filepath.Base(configFilePath)
dir := filepath.Dir(configFilePath)
if dir == configName && dir == "." {
configName = "default"
}
configDir, _ := filepath.Abs(dir)
viper.SetConfigName(filepath.Base(configFilePath))
viper.AddConfigPath(configDir) viper.AddConfigPath(configDir)
viper.SetConfigName(configName)
err := viper.ReadInConfig() // Find and read the config file err := viper.ReadInConfig()
if err != nil { // Handle errors reading the config file if err != nil { // Handle errors reading the config file
panic(fmt.Errorf("fatal error config file: %w", err)) panic(fmt.Errorf("fatal error config file: %w", err))
} }
viper.SetDefault("ConfigDir", configDir) viper.SetConfigName("secret")
scope := viper.GetString("scope") err = viper.MergeInConfig()
if scope != "" { if err != nil { // Handle errors reading the config file
viper.SetConfigName(scope) panic(fmt.Errorf("fatal error secret file: %w", err))
viper.AddConfigPath(configDir) }
err := viper.MergeInConfig()
if err != nil { slog.Info("using config", "path", filepath.Join(configDir, configName))
log.Fatalf("fatal error config file: %v", err) viper.SetDefault(constant.CfgKeyConfigDir, configDir)
return
// Scopes validating
scope := viper.GetString(constant.CfgKeyScopeEnable)
scopesAllowed := viper.GetStringSlice("scope.allow")
if !slices.Contains(scopesAllowed, scope) {
scope = viper.GetString("scope.default")
if scope == "" {
scope = constant.ScopeInfo
} }
} }
fmt.Println("scope", scope)
viper.SetConfigName(scope)
viper.AddConfigPath(filepath.Join(configDir, "scope"))
err = viper.MergeInConfig()
if err == nil {
viper.SetDefault(constant.CfgKeyScopeEnable, scope)
} else {
log.Fatalf("fatal error config file: %v", err)
}
} }

23
internal/flag.go
View File

@ -8,6 +8,8 @@ import (
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/spf13/viper" "github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
) )
const usage = `Music Database (MDB) server/cli craftware' const usage = `Music Database (MDB) server/cli craftware'
@ -25,19 +27,18 @@ The Go runtime version: %s
Report bugs to https://git.amok.space/yevhen/resource-scraper/issues` Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
const ( const (
maxPasswordLength = 20
version = "0.1" version = "0.1"
defaultConfigPath = "./config/default" defaultConfigPath = "config/default"
) )
func ParseFlags() { func ParseFlags() {
flag.Bool("h", false, "") flag.Bool(constant.FlagHelpShort, false, "")
flag.Bool("help", false, "") flag.Bool(constant.FlagHelp, false, "")
flag.Bool("v", false, "") flag.Bool(constant.FlagVersionShort, false, "")
flag.Bool("version", false, "") flag.Bool(constant.FlagVersion, false, "")
flag.Bool("debug", false, "") flag.Bool(constant.FlagDebug, false, "")
flag.String("config-file", defaultConfigPath, "config file location used for the program") flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location used for the program")
flag.String("scope", "", "") flag.String(constant.FlagScopeEnable, "", "")
pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
pflag.Parse() pflag.Parse()
@ -46,10 +47,10 @@ func ParseFlags() {
log.Fatalf("[ERR] Failed to bind flags to config: %s", err) log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
} }
if viper.GetBool("h") || viper.GetBool("help") { if viper.GetBool(constant.FlagHelpShort) || viper.GetBool(constant.FlagHelp) {
msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version()) msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
fmt.Println(msg) fmt.Println(msg)
} else if viper.GetBool("v") || viper.GetBool("version") { } else if viper.GetBool(constant.FlagVersionShort) || viper.GetBool(constant.FlagVersion) {
fmt.Println("MDB version", version) fmt.Println("MDB version", version)
} }
} }

59
logpresece Normal file
View File

@ -0,0 +1,59 @@
scope prescene
FLOAT: 33.000000, 12.000000, 3
false
====================== 1 ==============================
{Id:825647 Type:prescene TypeId:4313240 Title:Phaethon-Wielder_Of_The_Steel-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/phaethon-wielder_of_the_steel-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:09:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 2 ==============================
{Id:825648 Type:prescene TypeId:4313238 Title:Helvetets_Port-Warlords-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/helvetets_port-warlords-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:08:33 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 4 ==============================
{Id:825649 Type:prescene TypeId:4313236 Title:Circle_of_Ouroborus-Ajattoman_Uni-FI-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/circle_of_ouroborus-ajattoman_uni-fi-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:07:20 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 5 ==============================
{Id:825650 Type:prescene TypeId:4313234 Title:Blitzkrieg-Blitzkrieg-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/blitzkrieg-blitzkrieg-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:06:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 6 ==============================
{Id:825651 Type:prescene TypeId:4313233 Title:Antioch-Onward_With_Obliteration-EP-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/antioch-onward_with_obliteration-ep-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:05:27 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 8 ==============================
{Id:825631 Type:prescene TypeId:4313221 Title:Wretchgod-Suffering_Upon_Suffering-EP-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-suffering_upon_suffering-ep-24bit-web-flac-2024-moonblood/ Created:2024-09-10 14:07:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 9 ==============================
{Id:825632 Type:prescene TypeId:4313220 Title:Wretchgod-Stygian_Blood_Ritual-EP-16BIT-WEB-FLAC-2022-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-stygian_blood_ritual-ep-16bit-web-flac-2022-moonblood/ Created:2024-09-10 14:06:51 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 11 ==============================
{Id:825633 Type:prescene TypeId:4313188 Title:1349-The_Infernal_Pathway-REPACK-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-repack-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:34:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 15 ==============================
{Id:825634 Type:prescene TypeId:4313177 Title:1349-The_Infernal_Pathway-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:17:42 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 16 ==============================
{Id:825635 Type:prescene TypeId:4313175 Title:1349-Revelations_of_the_Black_Flame-16BIT-WEB-FLAC-2009-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-revelations_of_the_black_flame-16bit-web-flac-2009-moonblood/ Created:2024-09-10 13:16:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 17 ==============================
{Id:825636 Type:prescene TypeId:4313173 Title:1349-Massive_Cauldron_of_Chaos-16BIT-WEB-FLAC-2014-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-massive_cauldron_of_chaos-16bit-web-flac-2014-moonblood/ Created:2024-09-10 13:15:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 19 ==============================
{Id:825637 Type:prescene TypeId:4313171 Title:1349-Liberation-16BIT-WEB-FLAC-2003-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-liberation-16bit-web-flac-2003-moonblood/ Created:2024-09-10 13:14:38 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 21 ==============================
{Id:825638 Type:prescene TypeId:4313168 Title:1349-Hellfire-16BIT-WEB-FLAC-2005-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-hellfire-16bit-web-flac-2005-moonblood/ Created:2024-09-10 13:13:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 22 ==============================
{Id:825639 Type:prescene TypeId:4313166 Title:1349-Demonoir-REISSUE-16BIT-WEB-FLAC-2011-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-demonoir-reissue-16bit-web-flac-2011-moonblood/ Created:2024-09-10 13:12:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 23 ==============================
{Id:825640 Type:prescene TypeId:4313164 Title:1349-Beyond_the_Apocalypse-16BIT-WEB-FLAC-2004-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-beyond_the_apocalypse-16bit-web-flac-2004-moonblood/ Created:2024-09-10 13:12:00 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
Sleeping... 0
false
====================== 1 ==============================
{Id:825652 Type:prescene TypeId:4300530 Title:Zuckerbaby--Zuckerbaby-CD-FLAC-1997-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/zuckerbaby-zuckerbaby-cd-flac-1997-meowzik/ Created:2024-08-31 18:17:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 2 ==============================
{Id:825653 Type:prescene TypeId:4300529 Title:Silmarils--Vegas_76-FR-CD-FLAC-2000-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/silmarils-vegas_76-fr-cd-flac-2000-meowzik/ Created:2024-08-31 18:16:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 3 ==============================
{Id:825654 Type:prescene TypeId:4300527 Title:Michael_Kaeshammer--Days_Like_These-CD-FLAC-2007-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/michael_kaeshammer-days_like_these-cd-flac-2007-meowzik/ Created:2024-08-31 18:14:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 4 ==============================
{Id:825655 Type:prescene TypeId:4300521 Title:Butch_Walker--Letters-CD-FLAC-2004-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/butch_walker-letters-cd-flac-2004-meowzik/ Created:2024-08-31 18:09:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 16 ==============================
{Id:825656 Type:prescene TypeId:4300320 Title:Embodiment_Elimination-Metamorphosis_Incarnate_Through_Genetic_Devastation-(ISR071-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/embodiment_elimination-metamorphosis_incarnate_through_genetic_devastation-isr071-24-cd-flac-2024-86d/ Created:2024-08-31 11:19:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 19 ==============================
{Id:825657 Type:prescene TypeId:4300309 Title:Putrified_Degradation-Manufractured-(ISR072-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/putrified_degradation-manufractured-isr072-24-cd-flac-2024-86d/ Created:2024-08-31 10:45:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 20 ==============================
{Id:825658 Type:prescene TypeId:4300307 Title:Post_Mortal_Possession-The_Dead_Space_Between_the_Stars-(LORD091)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/post_mortal_possession-the_dead_space_between_the_stars-lord091-cd-flac-2024-86d/ Created:2024-08-31 10:43:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 21 ==============================
{Id:825659 Type:prescene TypeId:4300306 Title:Invirulant-Necrothuggin-(ISR070-24)-CDEP-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/invirulant-necrothuggin-isr070-24-cdep-flac-2024-86d/ Created:2024-08-31 10:41:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
====================== 22 ==============================
{Id:825660 Type:prescene TypeId:4300304 Title:Dead_and_Dripping-Disillusioned_by_Excessive_Human_Consumption-(LORD092)-DEMO_REISSUE-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/dead_and_dripping-disillusioned_by_excessive_human_consumption-lord092-demo_reissue-cd-flac-2024-86d/ Created:2024-08-31 10:38:13 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
Sleeping... 1
scope: prescene
[1 45]
init console console: ps
Execution took 12.6276275s

26
pkg/handler/handler.go
View File

@ -1,7 +1,10 @@
package handler package handler
import ( import (
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/pkg/service" "git.amok.space/yevhen/resource-scraper/pkg/service"
"git.amok.space/yevhen/resource-scraper/types/constant"
) )
type Handler struct { type Handler struct {
@ -13,16 +16,17 @@ func New(services *service.Service) *Handler {
} }
func (h *Handler) InitConsole() string { func (h *Handler) InitConsole() string {
return h.rutracker() switch viper.GetString(constant.CfgKeyScopeEnable) {
case constant.ScopeRuTracker:
return h.rutracker()
case constant.ScopePrescene:
return h.prescene()
}
return "no scope chosen"
} }
//func (h *Handler) Base(services *service.Service) *Handler { func (h *Handler) InitRoutes() string {
// return &Handler{services: services} //TODO:
//} return "i am the web initiator"
}
//func (h *Handler) InitApi() *chi.Mux {
// api := chi.NewRouter()
// api.Get("/", web.ApiFallbackHandler)
//
// return api
//}

46
pkg/handler/prescene.go Normal file
View File

@ -0,0 +1,46 @@
package handler
import (
"fmt"
"log/slog"
"math"
"strconv"
"time"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
)
func (h *Handler) prescene() string {
pagesToScrape := []string{"1"}
levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
if levels > 1 {
var q = (60.0 / 100.0) * (100.0 / viper.GetFloat64(constant.CfgKeyLevelsToScrape))
minute := float64(time.Now().Minute())
if minute < 1 {
minute = 1
}
fmt.Printf("FLOAT: %f, %f, %v\n", minute, q, math.Ceil(minute/q))
hour := time.Now().Hour()
if hour == 1 {
hour = 25
} else if hour == 0 {
hour = 1
}
page := hour * int(math.Ceil(minute/q))
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
}
_, err := h.services.Prescene.GetPage(pagesToScrape)
if err != nil {
slog.Error("error occurred while getting page: ", "err", err.Error())
}
fmt.Printf("%+v\n", pagesToScrape)
return "ps"
}

6
pkg/handler/rutracker.go
View File

@ -12,10 +12,12 @@ func (h *Handler) rutracker() string {
key := fmt.Sprintf("topic.%v", time.Now().Hour()) key := fmt.Sprintf("topic.%v", time.Now().Hour())
topics := viper.GetStringSlice(key) topics := viper.GetStringSlice(key)
err := h.services.Rutracker.GetTopic(topics) _, err := h.services.Rutracker.GetTopic(topics)
if err != nil { if err != nil {
slog.Error("error occurred while getting topic: ", err.Error()) slog.Error("error occurred while getting topic: ", "err", err.Error())
} }
//fmt.Printf("%+v", rt)
return "rt" return "rt"
} }

108
pkg/repository/prescene.go Normal file
View File

@ -0,0 +1,108 @@
package repository
import (
"fmt"
"log/slog"
"regexp"
"slices"
"strconv"
"time"
"github.com/go-shiori/dom"
"github.com/golang-module/carbon/v2"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/helper/parser"
"git.amok.space/yevhen/resource-scraper/pkg/repository/table"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.amok.space/yevhen/resource-scraper/types/model"
)
type Prescene struct {
db *sqlx.DB
}
func NewPresceneRepository(db *sqlx.DB) *Prescene {
return &Prescene{db: db}
}
func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
entries := make([]model.ExternalSources, 0)
endpoint := viper.GetString(constant.CfgKeyEndpoint)
//scope := viper.GetString(constant.CfgKeyScopeEnable)
tags := viper.GetStringMapStringSlice("groups.tags")
for _, t := range pageNumbers {
if t != "1" {
endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
}
doc, err := parser.HTMLSourceFromURL(endpoint)
//doc, err := parser.HTMLSourceFromURL("https://mdb.amok.space/$/scnlog.html")
if err != nil {
slog.Error("Parse error", "err", err)
continue
}
if doc == nil {
slog.Warn("Document is nil", "err", err)
continue
}
for _, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
var es model.ExternalSources
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
title := dom.QuerySelector(item, ".title")
if title != nil {
anchor := dom.QuerySelector(title, "h1 > a")
if anchor != nil {
es.Type = constant.ScopePrescene
es.Title = dom.GetAttribute(anchor, "title")
es.ExSource = dom.GetAttribute(anchor, "href")
pattern := regexp.MustCompile(`(?is)-(\w+)$`)
es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
for flag, groups := range tags {
if slices.Contains(groups, es.Releaser) {
es.A = flag
es.H = flag
columns = append(columns, "a", "h")
break
}
}
}
if es.A == constant.TagIgnore {
slog.Info("Skipped", "releaser", es.Releaser)
continue
}
localtime := dom.QuerySelector(title, "small > span.localtime")
if localtime != nil {
lc := dom.GetAttribute(localtime, "data-lttime")
es.Created = carbon.Parse(lc)
}
}
cls := dom.GetAttribute(item, "class")
pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
//fmt.Println("====================== ", i, " ==============================")
esModel := table.ExternalSources{Columns: columns}
entry := esModel.InsertOnDuplicate(es, s.db)
entries = append(entries, entry)
//fmt.Printf("%+v\n", entry)
}
//fmt.Println("Sleeping...", j)
time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
}
//fmt.Printf("scope: %v\n", scope)
return entries, nil
}

16
pkg/repository/repository.go
View File

@ -2,22 +2,18 @@ package repository
import ( import (
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types" "git.amok.space/yevhen/resource-scraper/types/interface"
) )
type Repository struct { type Repository struct {
types.Rutracker _interface.Rutracker
_interface.Prescene
} }
func New(db *sqlx.DB) *Repository { func New(db *sqlx.DB) *Repository {
switch viper.GetString("scope") { return &Repository{
case types.RuTracker: Rutracker: NewRutrackerRepository(db),
return &Repository{ Prescene: NewPresceneRepository(db),
Rutracker: NewRutracker(db),
}
} }
return &Repository{}
} }

47
pkg/repository/rutracker.go
View File

@ -8,60 +8,59 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"strconv" "strconv"
"time"
"github.com/golang-module/carbon/v2"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/spf13/viper" "github.com/spf13/viper"
iface "git.amok.space/yevhen/resource-scraper/types" "git.amok.space/yevhen/resource-scraper/pkg/repository/table"
"git.amok.space/yevhen/resource-scraper/types/table" "git.amok.space/yevhen/resource-scraper/types/constant"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.amok.space/yevhen/resource-scraper/types/resource"
) )
type Rutracker struct { type Rutracker struct {
db *sqlx.DB db *sqlx.DB
} }
func NewRutracker(db *sqlx.DB) *Rutracker { func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
return &Rutracker{db: db} return &Rutracker{db: db}
} }
func (s *Rutracker) GetTopic(topics []string) error { func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
endpoint := viper.GetString("endpoint") endpoint := viper.GetString(constant.CfgKeyEndpoint)
entries := make([]model.ExternalSources, 0)
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
for _, t := range topics { for _, t := range topics {
topic, err := fetch(fmt.Sprintf(endpoint, t)) topic, err := fetch(fmt.Sprintf(endpoint, t))
if err != nil { if err != nil {
slog.Error("couldn't parse topic data", "err", err.Error()) slog.Error("couldn't parse topic data", "err", err.Error())
return entries, err
} }
for i, e := range topic.Entry { for _, e := range topic.Entry {
var id int var es model.ExternalSources
var es table.ExternalSources
u, _ := url.Parse(e.Link.Href) u, _ := url.Parse(e.Link.Href)
es.Type = "rutracker"
es.Type = constant.ScopeRuTracker
es.TypeId, _ = strconv.Atoi(u.Query().Get("t")) es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
es.Title = e.Title es.Title = e.Title
es.TypeSubsectionId, _ = strconv.Atoi(t) es.TypeSubsectionId, _ = strconv.Atoi(t)
es.Releaser = e.Author.Name es.Releaser = e.Author.Name
es.Created, _ = time.Parse(time.RFC3339, e.Updated) es.Created = carbon.Parse(e.Updated)
created := es.Created.Format(iface.DateTimeFormat)
query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable) esModel := table.ExternalSources{Columns: columns}
entry := esModel.InsertOnDuplicate(es, s.db)
row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created) entries = append(entries, entry)
if err = row.Scan(&id); err != nil { //fmt.Printf("%+v\n\n\n", entry)
return err
}
fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>")
} }
} }
return nil return entries, nil
} }
func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) { func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
resp, err := http.Get(endpoint) resp, err := http.Get(endpoint)
if err != nil { if err != nil {
slog.Error("couldn't fetch data", endpoint, err.Error()) slog.Error("couldn't fetch data", endpoint, err.Error())
@ -75,7 +74,7 @@ func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
} }
}(resp.Body) }(resp.Body)
topic := &iface.RutrackerAtomTopic{} topic := &resource.RutrackerAtomTopic{}
if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil { if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
return nil, err return nil, err

34
pkg/repository/table/external_sources.go Normal file
View File

@ -0,0 +1,34 @@
package table
import (
"fmt"
"strings"
"github.com/jmoiron/sqlx"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.amok.space/yevhen/resource-scraper/types/model"
)
type ExternalSources struct {
Columns []string
}
func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.DB) model.ExternalSources {
stmt := "INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE title=:title, created=:created RETURNING id"
placeholders := strings.Join(f.Columns, ", :")
placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
//fmt.Printf("%s\n", query)
if rows, err := db.NamedQuery(query, &es); err == nil {
for rows.Next() {
es.Error = rows.StructScan(&es)
}
} else {
es.Error = err
}
return es
}

18
pkg/service/prescene.go Normal file
View File

@ -0,0 +1,18 @@
package service
import (
"git.amok.space/yevhen/resource-scraper/types/interface"
"git.amok.space/yevhen/resource-scraper/types/model"
)
type PresceneService struct {
repo _interface.Prescene
}
func NewPresceneService(repo _interface.Prescene) *PresceneService {
return &PresceneService{repo: repo}
}
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
return p.repo.GetPage(pageNumbers)
}

9
pkg/service/rutracker.go
View File

@ -1,17 +1,18 @@
package service package service
import ( import (
iface "git.amok.space/yevhen/resource-scraper/types" "git.amok.space/yevhen/resource-scraper/types/interface"
"git.amok.space/yevhen/resource-scraper/types/model"
) )
type RutrackerService struct { type RutrackerService struct {
repo iface.Rutracker repo _interface.Rutracker
} }
func NewRutrackerService(repo iface.Rutracker) *RutrackerService { func NewRutrackerService(repo _interface.Rutracker) *RutrackerService {
return &RutrackerService{repo: repo} return &RutrackerService{repo: repo}
} }
func (s *RutrackerService) GetTopic(topic []string) error { func (s *RutrackerService) GetTopic(topic []string) ([]model.ExternalSources, error) {
return s.repo.GetTopic(topic) return s.repo.GetTopic(topic)
} }

18
pkg/service/service.go
View File

@ -1,24 +1,18 @@
package service package service
import ( import (
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/pkg/repository" "git.amok.space/yevhen/resource-scraper/pkg/repository"
"git.amok.space/yevhen/resource-scraper/types" "git.amok.space/yevhen/resource-scraper/types/interface"
) )
type Service struct { type Service struct {
types.Rutracker _interface.Rutracker
_interface.Prescene
} }
func New(repos *repository.Repository) *Service { func New(repos *repository.Repository) *Service {
return &Service{
switch viper.GetString("scope") { Rutracker: NewRutrackerService(repos.Rutracker),
case types.RuTracker: Prescene: NewPresceneService(repos.Prescene),
return &Service{
Rutracker: NewRutrackerService(repos.Rutracker),
}
} }
return &Service{}
} }

10
types/constant/config.go Normal file
View File

@ -0,0 +1,10 @@
package constant
const (
CfgKeyScopeEnable = FlagScopeEnable
CfgKeyConfigDir = "config-dir"
CfgKeySleepBeforeNextIteration = "sleep-before-next-iteration"
CfgKeyEndpoint = "endpoint"
CfgKeyEndpointNext = "endpoint-next"
CfgKeyLevelsToScrape = "levels-to-scrape"
)

18
types/constant/constants.go Normal file
View File

@ -0,0 +1,18 @@
package constant
const (
ScopeRuTracker string = "rutracker"
ScopePrescene string = "prescene"
ScopeWeb string = "web"
ScopeInfo string = "info"
)
const (
RoleConsole string = "console"
RoleWeb string = "web"
)
const (
TagIgnore string = "ignore"
ExternalSourcesTable string = "external_sources"
)

11
types/constant/flag.go Normal file
View File

@ -0,0 +1,11 @@
package constant
const (
FlagHelp = "help"
FlagHelpShort = "h"
FlagDebug = "debug"
FlagVersion = "version"
FlagVersionShort = "v"
FlagConfigFile = "config-file"
FlagScopeEnable = "scope-enable"
)

14
types/constants.go
View File

@ -1,14 +0,0 @@
package types
const (
RuTracker string = "rutracker"
)
const (
RoleConsole string = "console"
)
const (
DateTimeFormat string = "2006-01-02 15:04:05"
ExternalSourcesTable string = "external_sources"
)

11
types/interface/interfaces.go Normal file
View File

@ -0,0 +1,11 @@
package _interface
import "git.amok.space/yevhen/resource-scraper/types/model"
type Rutracker interface {
GetTopic(topics []string) ([]model.ExternalSources, error)
}
type Prescene interface {
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
}

5
types/interfaces.go
View File

@ -1,5 +0,0 @@
package types
type Rutracker interface {
GetTopic(topics []string) error
}

42
types/model/external_sources.go Normal file
View File

@ -0,0 +1,42 @@
package model
import (
"github.com/golang-module/carbon/v2"
)
type Type string
/*const (
mmt Type = iota
prescene
rutracker
locate
darkabyss
bfm
trash
nnmc
stb
bandcamp
deathgrind
)*/
type ExternalSources struct {
Id int `json:"id" db:"id"`
Type string `json:"type" db:"type"`
TypeId int `json:"type_id" db:"type_id"`
Title string `json:"title" db:"title"`
TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
Releaser string `json:"releaser" db:"releaser"`
ExSource string `json:"ex_source" db:"eXsource"`
Created carbon.Carbon `json:"created_at" db:"created"`
Modified carbon.Carbon `json:"modified_at" db:"modified"`
Contents string `json:"contents" db:"contents"`
A string `json:"a" db:"a"`
H string `json:"h" db:"h"`
Fingerprint string `json:"fingerprint" db:"fingerprint"`
FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
Vid int `json:"vid" db:"vid"`
G string `json:"g" db:"g"`
ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
Error error
}

2
types/rutracker.go → types/resource/rutracker.go
View File

@ -1,4 +1,4 @@
package types package resource
import "encoding/xml" import "encoding/xml"

46
types/resource/scnlog.go Normal file
View File

@ -0,0 +1,46 @@
package resource
import "encoding/xml"
type ScnLogRss struct {
XMLName xml.Name `xml:"rss"`
Text string `xml:",chardata"`
Version string `xml:"version,attr"`
Content string `xml:"content,attr"`
Wfw string `xml:"wfw,attr"`
Dc string `xml:"dc,attr"`
Atom string `xml:"atom,attr"`
Sy string `xml:"sy,attr"`
Slash string `xml:"slash,attr"`
Channel struct {
Text string `xml:",chardata"`
Title string `xml:"title"`
Link struct {
Text string `xml:",chardata"`
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
} `xml:"link"`
Description string `xml:"description"`
LastBuildDate string `xml:"lastBuildDate"`
Language string `xml:"language"`
UpdatePeriod string `xml:"updatePeriod"`
UpdateFrequency string `xml:"updateFrequency"`
Generator string `xml:"generator"`
Item []struct {
Text string `xml:",chardata"`
Title string `xml:"title"`
Link string `xml:"link"`
Comments string `xml:"comments"`
Creator string `xml:"creator"`
PubDate string `xml:"pubDate"`
Category []string `xml:"category"`
Guid struct {
Text string `xml:",chardata"`
IsPermaLink string `xml:"isPermaLink,attr"`
} `xml:"guid"`
Description string `xml:"description"`
CommentRss string `xml:"commentRss"`
} `xml:"item"`
} `xml:"channel"`
}

39
types/table/external_sources.go
View File

@ -1,39 +0,0 @@
package table
import "time"
type Type string
/*const (
mmt Type = iota
prescene
rutracker
locate
darkabyss
bfm
trash
nnmc
stb
bandcamp
deathgrind
)*/
type ExternalSources struct {
Id int `json:"id" db:"id"`
Type string `json:"type" db:"type"`
TypeId int `json:"type_id" db:"type_id"`
Title string `json:"title" db:"title"`
TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
Releaser string `json:"releaser" db:"releaser"`
ExSource string `json:"ex_source" db:"eXsource"`
Created time.Time `json:"created_at" db:"created"`
Modified time.Time `json:"modified_at" db:"modified"`
Contents string `json:"contents" db:"contents"`
A string `json:"a" db:"a"`
H string `json:"h" db:"h"`
Fingerprint string `json:"fingerprint" db:"fingerprint"`
FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
Vid int `json:"vid" db:"vid"`
G string `json:"g" db:"g"`
ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
}