diff --git a/.air.toml b/.air.toml
index 0941617..a0815d3 100644
--- a/.air.toml
+++ b/.air.toml
@@ -4,7 +4,7 @@ tmp_dir = "tmp"
[build]
args_bin = []
- bin = "tmp\\main.exe --config-file ./config/default --scope rutracker"
+ bin = "tmp\\main.exe --config-file config/default --scope rutracker"
cmd = "go build -o ./tmp/main.exe ./cmd"
delay = 1000
exclude_dir = ["assets", "tmp", "vendor", "testdata"]
diff --git a/.gitignore b/.gitignore
index 903d241..4d32dcb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,4 +31,4 @@ fabric.properties
.env
tmp/
vendor/
-config/*.yaml
\ No newline at end of file
+config/secret.yaml
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
index ddc2326..53ae3b9 100644
--- a/.idea/inspectionProfiles/Project_Default.xml
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -3,7 +3,7 @@
-
+
diff --git a/cmd/main.go b/cmd/main.go
index e136c02..8da4279 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -1,16 +1,42 @@
package main
import (
+ "fmt"
+ "time"
+
_ "github.com/go-sql-driver/mysql"
scraper "git.amok.space/yevhen/resource-scraper/internal"
+ "git.amok.space/yevhen/resource-scraper/internal/config"
)
+var st time.Time
+
+//var tokenAuth *jwtauth.JWTAuth
+
func init() {
+ st = time.Now()
scraper.ParseFlags()
+ config.New()
+
+ /*verifyKey := []byte(viper.GetString("jwt.sign_key"))
+ tokenAuth = jwtauth.New("HS256", verifyKey, nil)
+
+ // For debugging/example purposes, we generate and print
+ // a sample jwt token with claims `user_id:123` here:
+ _, tokenString, _ := tokenAuth.Encode(map[string]interface{}{"user_id": 123})
+ slog.Info("jwt", "token", tokenString)*/
}
func main() {
+ defer func() {
+ fmt.Printf("\nExecution took %v", time.Since(st))
+ }()
+
+ //a, b := runtime.GOOS
+ //fmt.Printf("%v - - - \n", runtime.GOOS)
+
scraper.Bootstrap()
//fmt.Printf("%v\n", viper.Get("23").([]interface{}))
+ // Кінґстонська стіна
}
diff --git a/config/default.yaml b/config/default.yaml
new file mode 100644
index 0000000..8d09866
--- /dev/null
+++ b/config/default.yaml
@@ -0,0 +1,3 @@
+scope:
+ allow: [info, web, rutracker, prescene]
+ default: web
diff --git a/config/default.yml-sample b/config/default.yml-sample
deleted file mode 100644
index 976c17a..0000000
--- a/config/default.yml-sample
+++ /dev/null
@@ -1,4 +0,0 @@
-db:
- driver: mysql
- data_source_name: user:passwd@(localhost:3306)/dbname
-
diff --git a/config/scope/info.yaml b/config/scope/info.yaml
new file mode 100644
index 0000000..3fac415
--- /dev/null
+++ b/config/scope/info.yaml
@@ -0,0 +1 @@
+role: console
diff --git a/config/scope/prescene.yaml b/config/scope/prescene.yaml
new file mode 100644
index 0000000..e50f833
--- /dev/null
+++ b/config/scope/prescene.yaml
@@ -0,0 +1,69 @@
+role: console
+endpoint: https://scnlog.me/music/flac/
+endpoint-next: page/%s/
+sleep-before-next-iteration: 5s
+levels-to-scrape: 5
+groups:
+ tags:
+ neformat:
+ - SHGZ
+ remove:
+ - BIGLOVE
+ - CT
+ - ENViED
+ - GARLICKNOTS
+ - Ouzo
+ - PTCx
+ - SHGZ
+ - XiVERO
+ ignore:
+ - "401"
+ - 6DM
+ - AFO
+ - AOVF
+ - AOVF_Classics
+ - AUDiOFiLE
+ - BABAS
+ - BBD
+ - BEATOCUL
+ - CALiFLAC
+ - CEBAD
+ - D2H
+ - DALiAS
+ - dh
+ - DWM
+ - EMP
+ - ENRiCH
+ - ERP
+ - ESGFLAC
+ - FATHEAD
+ - FiXIE
+ - FLACME
+ - FrB
+ - HOUND
+ - JRO
+ - KINDA
+ - MAHOU
+ - MFDOS
+ - MLS
+ - Mrflac
+ - MUNDANE
+ - MyDad
+ - OBZEN
+ - OTT
+ - PERFECT
+ - PLAYLiST
+ - PTC
+ - PWT
+ - RAGEFLAC
+ - RAWBEATS
+ - RECTiFY
+ - ROSiN
+ - STAX
+ - THEVOiD
+ - TiMES
+ - TM
+ - TVRf
+ - VEXED
+ - WAVED
+ - YARD
diff --git a/config/rutracker.yml-sample b/config/scope/rutracker.yaml
similarity index 100%
rename from config/rutracker.yml-sample
rename to config/scope/rutracker.yaml
diff --git a/config/scope/web.yaml b/config/scope/web.yaml
new file mode 100644
index 0000000..8a3dcdb
--- /dev/null
+++ b/config/scope/web.yaml
@@ -0,0 +1,3 @@
+role: web
+http:
+ port: 19576
\ No newline at end of file
diff --git a/config/secret.yaml-sample b/config/secret.yaml-sample
new file mode 100644
index 0000000..7580094
--- /dev/null
+++ b/config/secret.yaml-sample
@@ -0,0 +1,6 @@
+db:
+ driver: mysql
+ data_source_name:
+jwt:
+ ttl: 6h12m24s
+ sign_key:
diff --git a/go.mod b/go.mod
index 862db12..bd9fa67 100644
--- a/go.mod
+++ b/go.mod
@@ -5,30 +5,46 @@ go 1.23.1
require (
github.com/go-chi/chi/v5 v5.1.0
github.com/go-chi/cors v1.2.1
+ github.com/go-chi/jwtauth/v5 v5.3.1
+ github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
github.com/go-sql-driver/mysql v1.8.1
+ github.com/golang-module/carbon/v2 v2.3.12
github.com/jmoiron/sqlx v1.4.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.19.0
+ golang.org/x/net v0.29.0
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
+ github.com/andybalholm/cascadia v1.3.2 // indirect
+ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
+ github.com/goccy/go-json v0.10.2 // indirect
+ github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
+ github.com/lestrrat-go/blackmagic v1.0.2 // indirect
+ github.com/lestrrat-go/httpcc v1.0.1 // indirect
+ github.com/lestrrat-go/httprc v1.0.4 // indirect
+ github.com/lestrrat-go/iter v1.0.2 // indirect
+ github.com/lestrrat-go/jwx/v2 v2.0.20 // indirect
+ github.com/lestrrat-go/option v1.0.1 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+ github.com/segmentio/asm v1.2.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
+ golang.org/x/crypto v0.27.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
- golang.org/x/sys v0.18.0 // indirect
- golang.org/x/text v0.14.0 // indirect
+ golang.org/x/sys v0.25.0 // indirect
+ golang.org/x/text v0.18.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index e736876..dd32e8c 100644
--- a/go.sum
+++ b/go.sum
@@ -1,9 +1,13 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -12,8 +16,18 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-chi/jwtauth/v5 v5.3.1 h1:1ePWrjVctvp1tyBq5b/2ER8Th/+RbYc7x4qNsc5rh5A=
+github.com/go-chi/jwtauth/v5 v5.3.1/go.mod h1:6Fl2RRmWXs3tJYE1IQGX81FsPoGqDwq9c15j52R5q80=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
+github.com/golang-module/carbon/v2 v2.3.12 h1:VC1DwN1kBwJkh5MjXmTFryjs5g4CWyoM8HAHffZPX/k=
+github.com/golang-module/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -24,6 +38,18 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
+github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
+github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
+github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
+github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8=
+github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
+github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
+github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
+github.com/lestrrat-go/jwx/v2 v2.0.20 h1:sAgXuWS/t8ykxS9Bi2Qtn5Qhpakw1wrcjxChudjolCc=
+github.com/lestrrat-go/jwx/v2 v2.0.20/go.mod h1:UlCSmKqw+agm5BsOBfEAbTvKsEApaGNqHAEUTv5PJC4=
+github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
+github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
@@ -43,6 +69,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
@@ -58,6 +86,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
@@ -65,16 +94,54 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/helper/parser/html.go b/helper/parser/html.go
new file mode 100644
index 0000000..e710c48
--- /dev/null
+++ b/helper/parser/html.go
@@ -0,0 +1,63 @@
+package parser
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "log/slog"
+ "net/http"
+ "strconv"
+ "strings"
+
+ "golang.org/x/net/html"
+)
+
+func setClient(url string) (*http.Response, error) {
+ client := &http.Client{}
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Fatalln(err)
+ }
+
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
+
+ return client.Do(req)
+}
+
+func HTMLSourceFromURL(url string) (*html.Node, error) {
+ resp, err := setClient(url)
+ defer func(Body io.ReadCloser) {
+ fmt.Printf("%v\n", Body == nil)
+
+ err = Body.Close()
+ if err != nil {
+ slog.Error("closing response body", "err", err)
+ }
+ }(resp.Body)
+
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode >= 400 {
+ return nil, errors.New("http status code: " + strconv.Itoa(resp.StatusCode))
+ }
+
+ doc, err := html.Parse(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ return doc, nil
+}
+
+func HTMLSource(htmlSource string) (*html.Node, error) {
+ doc, err := html.Parse(strings.NewReader(htmlSource))
+ if err != nil {
+ return nil, err
+ }
+ return doc, nil
+}
+
+//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
diff --git a/internal/bootstrap.go b/internal/bootstrap.go
index 2299c04..b4fcf0b 100644
--- a/internal/bootstrap.go
+++ b/internal/bootstrap.go
@@ -5,16 +5,15 @@ import (
"github.com/spf13/viper"
- "git.amok.space/yevhen/resource-scraper/internal/config"
"git.amok.space/yevhen/resource-scraper/internal/db"
"git.amok.space/yevhen/resource-scraper/pkg/handler"
"git.amok.space/yevhen/resource-scraper/pkg/repository"
"git.amok.space/yevhen/resource-scraper/pkg/service"
- "git.amok.space/yevhen/resource-scraper/types"
+ "git.amok.space/yevhen/resource-scraper/types/constant"
)
func Bootstrap() {
- config.New()
+ //https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
dbase := db.New()
repos := repository.New(dbase)
@@ -22,10 +21,12 @@ func Bootstrap() {
handlers := handler.New(services)
switch viper.GetString("role") {
- case types.RoleConsole:
+ case constant.RoleConsole:
fmt.Printf("init console console: %s\n", handlers.InitConsole())
break
+ case constant.RoleWeb:
+ fmt.Printf("who: %s\n", handlers.InitRoutes())
+ ///http.Run()
+ break
}
-
- ///http.Run()
}
diff --git a/internal/config/config.go b/internal/config/config.go
index 4609319..957059f 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -3,32 +3,62 @@ package config
import (
"fmt"
"log"
+ "log/slog"
"path/filepath"
+ "slices"
"github.com/spf13/viper"
+
+ "git.amok.space/yevhen/resource-scraper/types/constant"
)
func New() {
- configFilePath := viper.GetString("config-file")
- configDir := "./" + filepath.Dir(configFilePath)
+ configFilePath := viper.GetString(constant.FlagConfigFile)
+ configName := filepath.Base(configFilePath)
+
+ dir := filepath.Dir(configFilePath)
+ if dir == configName && dir == "." {
+ configName = "default"
+ }
+
+ configDir, _ := filepath.Abs(dir)
- viper.SetConfigName(filepath.Base(configFilePath))
viper.AddConfigPath(configDir)
-
- err := viper.ReadInConfig() // Find and read the config file
- if err != nil { // Handle errors reading the config file
+ viper.SetConfigName(configName)
+ err := viper.ReadInConfig()
+ if err != nil { // Handle errors reading the config file
panic(fmt.Errorf("fatal error config file: %w", err))
}
- viper.SetDefault("ConfigDir", configDir)
- scope := viper.GetString("scope")
- if scope != "" {
- viper.SetConfigName(scope)
- viper.AddConfigPath(configDir)
- err := viper.MergeInConfig()
- if err != nil {
- log.Fatalf("fatal error config file: %v", err)
- return
+ viper.SetConfigName("secret")
+ err = viper.MergeInConfig()
+ if err != nil { // Handle errors reading the config file
+ panic(fmt.Errorf("fatal error secret file: %w", err))
+ }
+
+ slog.Info("using config", "path", filepath.Join(configDir, configName))
+ viper.SetDefault(constant.CfgKeyConfigDir, configDir)
+
+ // Scopes validating
+ scope := viper.GetString(constant.CfgKeyScopeEnable)
+ scopesAllowed := viper.GetStringSlice("scope.allow")
+
+ if !slices.Contains(scopesAllowed, scope) {
+ scope = viper.GetString("scope.default")
+
+ if scope == "" {
+ scope = constant.ScopeInfo
}
}
+
+ fmt.Println("scope", scope)
+
+ viper.SetConfigName(scope)
+ viper.AddConfigPath(filepath.Join(configDir, "scope"))
+ err = viper.MergeInConfig()
+ if err == nil {
+ viper.SetDefault(constant.CfgKeyScopeEnable, scope)
+ } else {
+ log.Fatalf("fatal error config file: %v", err)
+ }
}
diff --git a/internal/flag.go b/internal/flag.go
index f7245a0..9420292 100644
--- a/internal/flag.go
+++ b/internal/flag.go
@@ -8,6 +8,8 @@ import (
"github.com/spf13/pflag"
"github.com/spf13/viper"
+
+ "git.amok.space/yevhen/resource-scraper/types/constant"
)
const usage = `Music Database (MDB) server/cli craftware'
@@ -25,19 +27,18 @@ The Go runtime version: %s
Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
const (
- maxPasswordLength = 20
version = "0.1"
- defaultConfigPath = "./config/default"
+ defaultConfigPath = "config/default"
)
func ParseFlags() {
- flag.Bool("h", false, "")
- flag.Bool("help", false, "")
- flag.Bool("v", false, "")
- flag.Bool("version", false, "")
- flag.Bool("debug", false, "")
- flag.String("config-file", defaultConfigPath, "config file location used for the program")
- flag.String("scope", "", "")
+ flag.Bool(constant.FlagHelpShort, false, "")
+ flag.Bool(constant.FlagHelp, false, "")
+ flag.Bool(constant.FlagVersionShort, false, "")
+ flag.Bool(constant.FlagVersion, false, "")
+ flag.Bool(constant.FlagDebug, false, "")
+ flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location used for the program")
+ flag.String(constant.FlagScopeEnable, "", "")
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
pflag.Parse()
@@ -46,10 +47,10 @@ func ParseFlags() {
log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
}
- if viper.GetBool("h") || viper.GetBool("help") {
+ if viper.GetBool(constant.FlagHelpShort) || viper.GetBool(constant.FlagHelp) {
msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
fmt.Println(msg)
- } else if viper.GetBool("v") || viper.GetBool("version") {
+ } else if viper.GetBool(constant.FlagVersionShort) || viper.GetBool(constant.FlagVersion) {
fmt.Println("MDB version", version)
}
}
diff --git a/logpresece b/logpresece
new file mode 100644
index 0000000..688c7d9
--- /dev/null
+++ b/logpresece
@@ -0,0 +1,59 @@
+scope prescene
+FLOAT: 33.000000, 12.000000, 3
+false
+====================== 1 ==============================
+{Id:825647 Type:prescene TypeId:4313240 Title:Phaethon-Wielder_Of_The_Steel-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/phaethon-wielder_of_the_steel-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:09:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 2 ==============================
+{Id:825648 Type:prescene TypeId:4313238 Title:Helvetets_Port-Warlords-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/helvetets_port-warlords-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:08:33 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 4 ==============================
+{Id:825649 Type:prescene TypeId:4313236 Title:Circle_of_Ouroborus-Ajattoman_Uni-FI-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/circle_of_ouroborus-ajattoman_uni-fi-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:07:20 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 5 ==============================
+{Id:825650 Type:prescene TypeId:4313234 Title:Blitzkrieg-Blitzkrieg-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/blitzkrieg-blitzkrieg-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:06:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 6 ==============================
+{Id:825651 Type:prescene TypeId:4313233 Title:Antioch-Onward_With_Obliteration-EP-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/antioch-onward_with_obliteration-ep-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:05:27 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 8 ==============================
+{Id:825631 Type:prescene TypeId:4313221 Title:Wretchgod-Suffering_Upon_Suffering-EP-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-suffering_upon_suffering-ep-24bit-web-flac-2024-moonblood/ Created:2024-09-10 14:07:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 9 ==============================
+{Id:825632 Type:prescene TypeId:4313220 Title:Wretchgod-Stygian_Blood_Ritual-EP-16BIT-WEB-FLAC-2022-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-stygian_blood_ritual-ep-16bit-web-flac-2022-moonblood/ Created:2024-09-10 14:06:51 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 11 ==============================
+{Id:825633 Type:prescene TypeId:4313188 Title:1349-The_Infernal_Pathway-REPACK-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-repack-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:34:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 15 ==============================
+{Id:825634 Type:prescene TypeId:4313177 Title:1349-The_Infernal_Pathway-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:17:42 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 16 ==============================
+{Id:825635 Type:prescene TypeId:4313175 Title:1349-Revelations_of_the_Black_Flame-16BIT-WEB-FLAC-2009-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-revelations_of_the_black_flame-16bit-web-flac-2009-moonblood/ Created:2024-09-10 13:16:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 17 ==============================
+{Id:825636 Type:prescene TypeId:4313173 Title:1349-Massive_Cauldron_of_Chaos-16BIT-WEB-FLAC-2014-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-massive_cauldron_of_chaos-16bit-web-flac-2014-moonblood/ Created:2024-09-10 13:15:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 19 ==============================
+{Id:825637 Type:prescene TypeId:4313171 Title:1349-Liberation-16BIT-WEB-FLAC-2003-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-liberation-16bit-web-flac-2003-moonblood/ Created:2024-09-10 13:14:38 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 21 ==============================
+{Id:825638 Type:prescene TypeId:4313168 Title:1349-Hellfire-16BIT-WEB-FLAC-2005-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-hellfire-16bit-web-flac-2005-moonblood/ Created:2024-09-10 13:13:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 22 ==============================
+{Id:825639 Type:prescene TypeId:4313166 Title:1349-Demonoir-REISSUE-16BIT-WEB-FLAC-2011-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-demonoir-reissue-16bit-web-flac-2011-moonblood/ Created:2024-09-10 13:12:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 23 ==============================
+{Id:825640 Type:prescene TypeId:4313164 Title:1349-Beyond_the_Apocalypse-16BIT-WEB-FLAC-2004-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-beyond_the_apocalypse-16bit-web-flac-2004-moonblood/ Created:2024-09-10 13:12:00 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+Sleeping... 0
+false
+====================== 1 ==============================
+{Id:825652 Type:prescene TypeId:4300530 Title:Zuckerbaby--Zuckerbaby-CD-FLAC-1997-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/zuckerbaby-zuckerbaby-cd-flac-1997-meowzik/ Created:2024-08-31 18:17:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 2 ==============================
+{Id:825653 Type:prescene TypeId:4300529 Title:Silmarils--Vegas_76-FR-CD-FLAC-2000-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/silmarils-vegas_76-fr-cd-flac-2000-meowzik/ Created:2024-08-31 18:16:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 3 ==============================
+{Id:825654 Type:prescene TypeId:4300527 Title:Michael_Kaeshammer--Days_Like_These-CD-FLAC-2007-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/michael_kaeshammer-days_like_these-cd-flac-2007-meowzik/ Created:2024-08-31 18:14:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 4 ==============================
+{Id:825655 Type:prescene TypeId:4300521 Title:Butch_Walker--Letters-CD-FLAC-2004-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/butch_walker-letters-cd-flac-2004-meowzik/ Created:2024-08-31 18:09:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 16 ==============================
+{Id:825656 Type:prescene TypeId:4300320 Title:Embodiment_Elimination-Metamorphosis_Incarnate_Through_Genetic_Devastation-(ISR071-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/embodiment_elimination-metamorphosis_incarnate_through_genetic_devastation-isr071-24-cd-flac-2024-86d/ Created:2024-08-31 11:19:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 19 ==============================
+{Id:825657 Type:prescene TypeId:4300309 Title:Putrified_Degradation-Manufractured-(ISR072-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/putrified_degradation-manufractured-isr072-24-cd-flac-2024-86d/ Created:2024-08-31 10:45:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 20 ==============================
+{Id:825658 Type:prescene TypeId:4300307 Title:Post_Mortal_Possession-The_Dead_Space_Between_the_Stars-(LORD091)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/post_mortal_possession-the_dead_space_between_the_stars-lord091-cd-flac-2024-86d/ Created:2024-08-31 10:43:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 21 ==============================
+{Id:825659 Type:prescene TypeId:4300306 Title:Invirulant-Necrothuggin-(ISR070-24)-CDEP-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/invirulant-necrothuggin-isr070-24-cdep-flac-2024-86d/ Created:2024-08-31 10:41:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+====================== 22 ==============================
+{Id:825660 Type:prescene TypeId:4300304 Title:Dead_and_Dripping-Disillusioned_by_Excessive_Human_Consumption-(LORD092)-DEMO_REISSUE-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/dead_and_dripping-disillusioned_by_excessive_human_consumption-lord092-demo_reissue-cd-flac-2024-86d/ Created:2024-08-31 10:38:13 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:}
+Sleeping... 1
+scope: prescene
+[1 45]
+init console console: ps
+
+Execution took 12.6276275s
\ No newline at end of file
diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go
index 7ca366b..0eca4c0 100644
--- a/pkg/handler/handler.go
+++ b/pkg/handler/handler.go
@@ -1,7 +1,10 @@
package handler
import (
+ "github.com/spf13/viper"
+
"git.amok.space/yevhen/resource-scraper/pkg/service"
+ "git.amok.space/yevhen/resource-scraper/types/constant"
)
type Handler struct {
@@ -13,16 +16,17 @@ func New(services *service.Service) *Handler {
}
func (h *Handler) InitConsole() string {
- return h.rutracker()
+ switch viper.GetString(constant.CfgKeyScopeEnable) {
+ case constant.ScopeRuTracker:
+ return h.rutracker()
+ case constant.ScopePrescene:
+ return h.prescene()
+ }
+
+ return "no scope chosen"
}
-//func (h *Handler) Base(services *service.Service) *Handler {
-// return &Handler{services: services}
-//}
-
-//func (h *Handler) InitApi() *chi.Mux {
-// api := chi.NewRouter()
-// api.Get("/", web.ApiFallbackHandler)
-//
-// return api
-//}
+func (h *Handler) InitRoutes() string {
+ //TODO:
+ return "i am the web initiator"
+}
diff --git a/pkg/handler/prescene.go b/pkg/handler/prescene.go
new file mode 100644
index 0000000..557e800
--- /dev/null
+++ b/pkg/handler/prescene.go
@@ -0,0 +1,46 @@
+package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "math"
+ "strconv"
+ "time"
+
+ "github.com/spf13/viper"
+
+ "git.amok.space/yevhen/resource-scraper/types/constant"
+)
+
+func (h *Handler) prescene() string {
+ pagesToScrape := []string{"1"}
+ levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
+ if levels > 1 {
+ var q = (60.0 / 100.0) * (100.0 / viper.GetFloat64(constant.CfgKeyLevelsToScrape))
+ minute := float64(time.Now().Minute())
+ if minute < 1 {
+ minute = 1
+ }
+
+ fmt.Printf("FLOAT: %f, %f, %v\n", minute, q, math.Ceil(minute/q))
+
+ hour := time.Now().Hour()
+ if hour == 1 {
+ hour = 25
+ } else if hour == 0 {
+ hour = 1
+ }
+
+ page := hour * int(math.Ceil(minute/q))
+ pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
+ }
+
+ _, err := h.services.Prescene.GetPage(pagesToScrape)
+ if err != nil {
+ slog.Error("error occurred while getting page: ", "err", err.Error())
+ }
+
+ fmt.Printf("%+v\n", pagesToScrape)
+
+ return "ps"
+}
diff --git a/pkg/handler/rutracker.go b/pkg/handler/rutracker.go
index 7729bb2..a116f8c 100644
--- a/pkg/handler/rutracker.go
+++ b/pkg/handler/rutracker.go
@@ -12,10 +12,12 @@ func (h *Handler) rutracker() string {
key := fmt.Sprintf("topic.%v", time.Now().Hour())
topics := viper.GetStringSlice(key)
- err := h.services.Rutracker.GetTopic(topics)
+ _, err := h.services.Rutracker.GetTopic(topics)
if err != nil {
- slog.Error("error occurred while getting topic: ", err.Error())
+ slog.Error("error occurred while getting topic: ", "err", err.Error())
}
+ //fmt.Printf("%+v", rt)
+
return "rt"
}
diff --git a/pkg/repository/prescene.go b/pkg/repository/prescene.go
new file mode 100644
index 0000000..0853b86
--- /dev/null
+++ b/pkg/repository/prescene.go
@@ -0,0 +1,108 @@
+package repository
+
+import (
+ "fmt"
+ "log/slog"
+ "regexp"
+ "slices"
+ "strconv"
+ "time"
+
+ "github.com/go-shiori/dom"
+ "github.com/golang-module/carbon/v2"
+ "github.com/jmoiron/sqlx"
+ "github.com/spf13/viper"
+
+ "git.amok.space/yevhen/resource-scraper/helper/parser"
+ "git.amok.space/yevhen/resource-scraper/pkg/repository/table"
+ "git.amok.space/yevhen/resource-scraper/types/constant"
+ "git.amok.space/yevhen/resource-scraper/types/model"
+)
+
+type Prescene struct {
+ db *sqlx.DB
+}
+
+func NewPresceneRepository(db *sqlx.DB) *Prescene {
+ return &Prescene{db: db}
+}
+
+func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
+ entries := make([]model.ExternalSources, 0)
+ endpoint := viper.GetString(constant.CfgKeyEndpoint)
+ //scope := viper.GetString(constant.CfgKeyScopeEnable)
+ tags := viper.GetStringMapStringSlice("groups.tags")
+
+ for _, t := range pageNumbers {
+ if t != "1" {
+ endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
+ }
+ doc, err := parser.HTMLSourceFromURL(endpoint)
+ //doc, err := parser.HTMLSourceFromURL("https://mdb.amok.space/$/scnlog.html")
+
+ if err != nil {
+ slog.Error("Parse error", "err", err)
+ continue
+ }
+
+ if doc == nil {
+ slog.Warn("Document is nil", "err", err)
+ continue
+ }
+
+ for _, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
+ var es model.ExternalSources
+ columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
+
+ title := dom.QuerySelector(item, ".title")
+ if title != nil {
+ anchor := dom.QuerySelector(title, "h1 > a")
+ if anchor != nil {
+ es.Type = constant.ScopePrescene
+ es.Title = dom.GetAttribute(anchor, "title")
+ es.ExSource = dom.GetAttribute(anchor, "href")
+ pattern := regexp.MustCompile(`(?is)-(\w+)$`)
+ es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
+
+ for flag, groups := range tags {
+ if slices.Contains(groups, es.Releaser) {
+ es.A = flag
+ es.H = flag
+ columns = append(columns, "a", "h")
+ break
+ }
+ }
+ }
+
+ if es.A == constant.TagIgnore {
+ slog.Info("Skipped", "releaser", es.Releaser)
+ continue
+ }
+
+ localtime := dom.QuerySelector(title, "small > span.localtime")
+ if localtime != nil {
+ lc := dom.GetAttribute(localtime, "data-lttime")
+ es.Created = carbon.Parse(lc)
+ }
+ }
+
+ cls := dom.GetAttribute(item, "class")
+ pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
+ es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
+
+ //fmt.Println("====================== ", i, " ==============================")
+ esModel := table.ExternalSources{Columns: columns}
+ entry := esModel.InsertOnDuplicate(es, s.db)
+ entries = append(entries, entry)
+
+ //fmt.Printf("%+v\n", entry)
+ }
+
+ //fmt.Println("Sleeping...", j)
+ time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
+ }
+
+ //fmt.Printf("scope: %v\n", scope)
+
+ return entries, nil
+}
diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go
index 94cb896..9d58274 100644
--- a/pkg/repository/repository.go
+++ b/pkg/repository/repository.go
@@ -2,22 +2,18 @@ package repository
import (
"github.com/jmoiron/sqlx"
- "github.com/spf13/viper"
- "git.amok.space/yevhen/resource-scraper/types"
+ "git.amok.space/yevhen/resource-scraper/types/interface"
)
type Repository struct {
- types.Rutracker
+ _interface.Rutracker
+ _interface.Prescene
}
func New(db *sqlx.DB) *Repository {
- switch viper.GetString("scope") {
- case types.RuTracker:
- return &Repository{
- Rutracker: NewRutracker(db),
- }
+ return &Repository{
+ Rutracker: NewRutrackerRepository(db),
+ Prescene: NewPresceneRepository(db),
}
-
- return &Repository{}
}
diff --git a/pkg/repository/rutracker.go b/pkg/repository/rutracker.go
index cf9b0f6..99152cd 100644
--- a/pkg/repository/rutracker.go
+++ b/pkg/repository/rutracker.go
@@ -8,60 +8,59 @@ import (
"net/http"
"net/url"
"strconv"
- "time"
+ "github.com/golang-module/carbon/v2"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
- iface "git.amok.space/yevhen/resource-scraper/types"
- "git.amok.space/yevhen/resource-scraper/types/table"
+ "git.amok.space/yevhen/resource-scraper/pkg/repository/table"
+ "git.amok.space/yevhen/resource-scraper/types/constant"
+ "git.amok.space/yevhen/resource-scraper/types/model"
+ "git.amok.space/yevhen/resource-scraper/types/resource"
)
type Rutracker struct {
db *sqlx.DB
}
-func NewRutracker(db *sqlx.DB) *Rutracker {
+func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
return &Rutracker{db: db}
}
-func (s *Rutracker) GetTopic(topics []string) error {
- endpoint := viper.GetString("endpoint")
+func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
+ endpoint := viper.GetString(constant.CfgKeyEndpoint)
+ entries := make([]model.ExternalSources, 0)
+ columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
for _, t := range topics {
topic, err := fetch(fmt.Sprintf(endpoint, t))
if err != nil {
slog.Error("couldn't parse topic data", "err", err.Error())
+ return entries, err
}
- for i, e := range topic.Entry {
- var id int
- var es table.ExternalSources
-
+ for _, e := range topic.Entry {
+ var es model.ExternalSources
u, _ := url.Parse(e.Link.Href)
- es.Type = "rutracker"
+
+ es.Type = constant.ScopeRuTracker
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
es.Title = e.Title
es.TypeSubsectionId, _ = strconv.Atoi(t)
es.Releaser = e.Author.Name
- es.Created, _ = time.Parse(time.RFC3339, e.Updated)
- created := es.Created.Format(iface.DateTimeFormat)
+ es.Created = carbon.Parse(e.Updated)
- query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable)
-
- row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created)
- if err = row.Scan(&id); err != nil {
- return err
- }
-
- fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>")
+ esModel := table.ExternalSources{Columns: columns}
+ entry := esModel.InsertOnDuplicate(es, s.db)
+ entries = append(entries, entry)
+ //fmt.Printf("%+v\n\n\n", entry)
}
}
- return nil
+ return entries, nil
}
-func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
+func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
resp, err := http.Get(endpoint)
if err != nil {
slog.Error("couldn't fetch data", endpoint, err.Error())
@@ -75,7 +74,7 @@ func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
}
}(resp.Body)
- topic := &iface.RutrackerAtomTopic{}
+ topic := &resource.RutrackerAtomTopic{}
if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
return nil, err
diff --git a/pkg/repository/table/external_sources.go b/pkg/repository/table/external_sources.go
new file mode 100644
index 0000000..a27b4a1
--- /dev/null
+++ b/pkg/repository/table/external_sources.go
@@ -0,0 +1,34 @@
+package table
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/jmoiron/sqlx"
+
+ "git.amok.space/yevhen/resource-scraper/types/constant"
+ "git.amok.space/yevhen/resource-scraper/types/model"
+)
+
+type ExternalSources struct {
+ Columns []string
+}
+
+func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.DB) model.ExternalSources {
+ stmt := "INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE title=:title, created=:created RETURNING id"
+ placeholders := strings.Join(f.Columns, ", :")
+ placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
+ query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
+
+ //fmt.Printf("%s\n", query)
+
+ if rows, err := db.NamedQuery(query, &es); err == nil {
+ for rows.Next() {
+ es.Error = rows.StructScan(&es)
+ }
+ } else {
+ es.Error = err
+ }
+
+ return es
+}
diff --git a/pkg/service/prescene.go b/pkg/service/prescene.go
new file mode 100644
index 0000000..03b1760
--- /dev/null
+++ b/pkg/service/prescene.go
@@ -0,0 +1,18 @@
+package service
+
+import (
+ "git.amok.space/yevhen/resource-scraper/types/interface"
+ "git.amok.space/yevhen/resource-scraper/types/model"
+)
+
+type PresceneService struct {
+ repo _interface.Prescene
+}
+
+func NewPresceneService(repo _interface.Prescene) *PresceneService {
+ return &PresceneService{repo: repo}
+}
+
+func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
+ return p.repo.GetPage(pageNumbers)
+}
diff --git a/pkg/service/rutracker.go b/pkg/service/rutracker.go
index 481f1c8..7194e38 100644
--- a/pkg/service/rutracker.go
+++ b/pkg/service/rutracker.go
@@ -1,17 +1,18 @@
package service
import (
- iface "git.amok.space/yevhen/resource-scraper/types"
+ "git.amok.space/yevhen/resource-scraper/types/interface"
+ "git.amok.space/yevhen/resource-scraper/types/model"
)
type RutrackerService struct {
- repo iface.Rutracker
+ repo _interface.Rutracker
}
-func NewRutrackerService(repo iface.Rutracker) *RutrackerService {
+func NewRutrackerService(repo _interface.Rutracker) *RutrackerService {
return &RutrackerService{repo: repo}
}
-func (s *RutrackerService) GetTopic(topic []string) error {
+func (s *RutrackerService) GetTopic(topic []string) ([]model.ExternalSources, error) {
return s.repo.GetTopic(topic)
}
diff --git a/pkg/service/service.go b/pkg/service/service.go
index 222b9d4..f547e7c 100644
--- a/pkg/service/service.go
+++ b/pkg/service/service.go
@@ -1,24 +1,18 @@
package service
import (
- "github.com/spf13/viper"
-
"git.amok.space/yevhen/resource-scraper/pkg/repository"
- "git.amok.space/yevhen/resource-scraper/types"
+ "git.amok.space/yevhen/resource-scraper/types/interface"
)
type Service struct {
- types.Rutracker
+ _interface.Rutracker
+ _interface.Prescene
}
func New(repos *repository.Repository) *Service {
-
- switch viper.GetString("scope") {
- case types.RuTracker:
- return &Service{
- Rutracker: NewRutrackerService(repos.Rutracker),
- }
+ return &Service{
+ Rutracker: NewRutrackerService(repos.Rutracker),
+ Prescene: NewPresceneService(repos.Prescene),
}
-
- return &Service{}
}
diff --git a/types/constant/config.go b/types/constant/config.go
new file mode 100644
index 0000000..1f8b96d
--- /dev/null
+++ b/types/constant/config.go
@@ -0,0 +1,10 @@
+package constant
+
+const (
+ CfgKeyScopeEnable = FlagScopeEnable
+ CfgKeyConfigDir = "config-dir"
+ CfgKeySleepBeforeNextIteration = "sleep-before-next-iteration"
+ CfgKeyEndpoint = "endpoint"
+ CfgKeyEndpointNext = "endpoint-next"
+ CfgKeyLevelsToScrape = "levels-to-scrape"
+)
diff --git a/types/constant/constants.go b/types/constant/constants.go
new file mode 100644
index 0000000..ba3260b
--- /dev/null
+++ b/types/constant/constants.go
@@ -0,0 +1,18 @@
+package constant
+
+const (
+ ScopeRuTracker string = "rutracker"
+ ScopePrescene string = "prescene"
+ ScopeWeb string = "web"
+ ScopeInfo string = "info"
+)
+
+const (
+ RoleConsole string = "console"
+ RoleWeb string = "web"
+)
+
+const (
+ TagIgnore string = "ignore"
+ ExternalSourcesTable string = "external_sources"
+)
diff --git a/types/constant/flag.go b/types/constant/flag.go
new file mode 100644
index 0000000..99b02bb
--- /dev/null
+++ b/types/constant/flag.go
@@ -0,0 +1,11 @@
+package constant
+
+const (
+ FlagHelp = "help"
+ FlagHelpShort = "h"
+ FlagDebug = "debug"
+ FlagVersion = "version"
+ FlagVersionShort = "v"
+ FlagConfigFile = "config-file"
+ FlagScopeEnable = "scope-enable"
+)
diff --git a/types/constants.go b/types/constants.go
deleted file mode 100644
index 7aa479c..0000000
--- a/types/constants.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package types
-
-const (
- RuTracker string = "rutracker"
-)
-
-const (
- RoleConsole string = "console"
-)
-
-const (
- DateTimeFormat string = "2006-01-02 15:04:05"
- ExternalSourcesTable string = "external_sources"
-)
diff --git a/types/interface/interfaces.go b/types/interface/interfaces.go
new file mode 100644
index 0000000..9a9353a
--- /dev/null
+++ b/types/interface/interfaces.go
@@ -0,0 +1,11 @@
+package _interface
+
+import "git.amok.space/yevhen/resource-scraper/types/model"
+
+type Rutracker interface {
+ GetTopic(topics []string) ([]model.ExternalSources, error)
+}
+
+type Prescene interface {
+ GetPage(pageNumbers []string) ([]model.ExternalSources, error)
+}
diff --git a/types/interfaces.go b/types/interfaces.go
deleted file mode 100644
index d7c7871..0000000
--- a/types/interfaces.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package types
-
-type Rutracker interface {
- GetTopic(topics []string) error
-}
diff --git a/types/model/external_sources.go b/types/model/external_sources.go
new file mode 100644
index 0000000..45cf29f
--- /dev/null
+++ b/types/model/external_sources.go
@@ -0,0 +1,42 @@
+package model
+
+import (
+ "github.com/golang-module/carbon/v2"
+)
+
+type Type string
+
+/*const (
+ mmt Type = iota
+ prescene
+ rutracker
+ locate
+ darkabyss
+ bfm
+ trash
+ nnmc
+ stb
+ bandcamp
+ deathgrind
+)*/
+
+type ExternalSources struct {
+ Id int `json:"id" db:"id"`
+ Type string `json:"type" db:"type"`
+ TypeId int `json:"type_id" db:"type_id"`
+ Title string `json:"title" db:"title"`
+ TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
+ Releaser string `json:"releaser" db:"releaser"`
+ ExSource string `json:"ex_source" db:"eXsource"`
+ Created carbon.Carbon `json:"created_at" db:"created"`
+ Modified carbon.Carbon `json:"modified_at" db:"modified"`
+ Contents string `json:"contents" db:"contents"`
+ A string `json:"a" db:"a"`
+ H string `json:"h" db:"h"`
+ Fingerprint string `json:"fingerprint" db:"fingerprint"`
+ FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
+ Vid int `json:"vid" db:"vid"`
+ G string `json:"g" db:"g"`
+ ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
+ Error error
+}
diff --git a/types/rutracker.go b/types/resource/rutracker.go
similarity index 98%
rename from types/rutracker.go
rename to types/resource/rutracker.go
index d00633e..cf2e1d2 100644
--- a/types/rutracker.go
+++ b/types/resource/rutracker.go
@@ -1,4 +1,4 @@
-package types
+package resource
import "encoding/xml"
diff --git a/types/resource/scnlog.go b/types/resource/scnlog.go
new file mode 100644
index 0000000..c269903
--- /dev/null
+++ b/types/resource/scnlog.go
@@ -0,0 +1,46 @@
+package resource
+
+import "encoding/xml"
+
+type ScnLogRss struct {
+ XMLName xml.Name `xml:"rss"`
+ Text string `xml:",chardata"`
+ Version string `xml:"version,attr"`
+ Content string `xml:"content,attr"`
+ Wfw string `xml:"wfw,attr"`
+ Dc string `xml:"dc,attr"`
+ Atom string `xml:"atom,attr"`
+ Sy string `xml:"sy,attr"`
+ Slash string `xml:"slash,attr"`
+ Channel struct {
+ Text string `xml:",chardata"`
+ Title string `xml:"title"`
+ Link struct {
+ Text string `xml:",chardata"`
+ Href string `xml:"href,attr"`
+ Rel string `xml:"rel,attr"`
+ Type string `xml:"type,attr"`
+ } `xml:"link"`
+ Description string `xml:"description"`
+ LastBuildDate string `xml:"lastBuildDate"`
+ Language string `xml:"language"`
+ UpdatePeriod string `xml:"updatePeriod"`
+ UpdateFrequency string `xml:"updateFrequency"`
+ Generator string `xml:"generator"`
+ Item []struct {
+ Text string `xml:",chardata"`
+ Title string `xml:"title"`
+ Link string `xml:"link"`
+ Comments string `xml:"comments"`
+ Creator string `xml:"creator"`
+ PubDate string `xml:"pubDate"`
+ Category []string `xml:"category"`
+ Guid struct {
+ Text string `xml:",chardata"`
+ IsPermaLink string `xml:"isPermaLink,attr"`
+ } `xml:"guid"`
+ Description string `xml:"description"`
+ CommentRss string `xml:"commentRss"`
+ } `xml:"item"`
+ } `xml:"channel"`
+}
diff --git a/types/table/external_sources.go b/types/table/external_sources.go
deleted file mode 100644
index 66408bc..0000000
--- a/types/table/external_sources.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package table
-
-import "time"
-
-type Type string
-
-/*const (
- mmt Type = iota
- prescene
- rutracker
- locate
- darkabyss
- bfm
- trash
- nnmc
- stb
- bandcamp
- deathgrind
-)*/
-
-type ExternalSources struct {
- Id int `json:"id" db:"id"`
- Type string `json:"type" db:"type"`
- TypeId int `json:"type_id" db:"type_id"`
- Title string `json:"title" db:"title"`
- TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
- Releaser string `json:"releaser" db:"releaser"`
- ExSource string `json:"ex_source" db:"eXsource"`
- Created time.Time `json:"created_at" db:"created"`
- Modified time.Time `json:"modified_at" db:"modified"`
- Contents string `json:"contents" db:"contents"`
- A string `json:"a" db:"a"`
- H string `json:"h" db:"h"`
- Fingerprint string `json:"fingerprint" db:"fingerprint"`
- FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
- Vid int `json:"vid" db:"vid"`
- G string `json:"g" db:"g"`
- ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
-}