preset scraper added

2024-09-10 15:57:48 +03:00 · 2024-09-10 15:57:48 +03:00 · 0ecf0ddec1
commit 0ecf0ddec1
parent bfdfc634e4
38 changed files with 790 additions and 167 deletions
--- a/.air.toml
+++ b/.air.toml
@ -4,7 +4,7 @@ tmp_dir = "tmp"

 [build]
  args_bin = []
-  bin = "tmp\\main.exe --config-file ./config/default --scope rutracker"
+  bin = "tmp\\main.exe --config-file config/default --scope rutracker"
  cmd = "go build -o ./tmp/main.exe ./cmd"
  delay = 1000
  exclude_dir = ["assets", "tmp", "vendor", "testdata"]
--- a/.gitignore
+++ b/.gitignore
@ -31,4 +31,4 @@ fabric.properties
 .env
 tmp/
 vendor/
-config/*.yaml
+config/secret.yaml
--- a/.idea/inspectionProfiles/Project_Default.xml
+++ b/.idea/inspectionProfiles/Project_Default.xml
@ -3,7 +3,7 @@
    <option name="myName" value="Project Default" />
    <inspection_tool class="GoDfaErrorMayBeNotNil" enabled="true" level="WARNING" enabled_by_default="true">
      <functions>
-        <function importPath="git.amok.space/yevhen/resource-scraper/pkg/repository" name="fetch" />
+        <function importPath="net/http" name="Get" />
      </functions>
    </inspection_tool>
  </profile>
--- a/cmd/main.go
+++ b/cmd/main.go
@ -1,16 +1,42 @@
 package main

 import (
+	"fmt"
+	"time"
+
 	_ "github.com/go-sql-driver/mysql"

 	scraper "git.amok.space/yevhen/resource-scraper/internal"
+	"git.amok.space/yevhen/resource-scraper/internal/config"
 )

+var st time.Time
+
+//var tokenAuth *jwtauth.JWTAuth
+
 func init() {
+	st = time.Now()
 	scraper.ParseFlags()
+	config.New()
+
+	/*verifyKey := []byte(viper.GetString("jwt.sign_key"))
+	tokenAuth = jwtauth.New("HS256", verifyKey, nil)
+
+	// For debugging/example purposes, we generate and print
+	// a sample jwt token with claims `user_id:123` here:
+	_, tokenString, _ := tokenAuth.Encode(map[string]interface{}{"user_id": 123})
+	slog.Info("jwt", "token", tokenString)*/
 }

 func main() {
+	defer func() {
+		fmt.Printf("\nExecution took %v", time.Since(st))
+	}()
+
+	//a, b := runtime.GOOS
+	//fmt.Printf("%v  - - -  \n", runtime.GOOS)
+
 	scraper.Bootstrap()
 	//fmt.Printf("%v\n", viper.Get("23").([]interface{}))
+	// Кінґстонська стіна
 }
--- a/config/default.yaml
+++ b/config/default.yaml
@ -0,0 +1,3 @@
+scope:
+  allow: [info, web, rutracker, prescene]
+  default: web
--- a/config/default.yml-sample
+++ b/config/default.yml-sample
@ -1,4 +0,0 @@
-db:
-  driver: mysql
-  data_source_name: user:passwd@(localhost:3306)/dbname
-
--- a/config/scope/info.yaml
+++ b/config/scope/info.yaml
@ -0,0 +1 @@
+role: console
--- a/config/scope/prescene.yaml
+++ b/config/scope/prescene.yaml
@ -0,0 +1,69 @@
+role: console
+endpoint: https://scnlog.me/music/flac/
+endpoint-next: page/%s/
+sleep-before-next-iteration: 5s
+levels-to-scrape: 5
+groups:
+  tags:
+    neformat:
+      - SHGZ
+    remove:
+      - BIGLOVE
+      - CT
+      - ENViED
+      - GARLICKNOTS
+      - Ouzo
+      - PTCx
+      - SHGZ
+      - XiVERO
+    ignore:
+      - "401"
+      - 6DM
+      - AFO
+      - AOVF
+      - AOVF_Classics
+      - AUDiOFiLE
+      - BABAS
+      - BBD
+      - BEATOCUL
+      - CALiFLAC
+      - CEBAD
+      - D2H
+      - DALiAS
+      - dh
+      - DWM
+      - EMP
+      - ENRiCH
+      - ERP
+      - ESGFLAC
+      - FATHEAD
+      - FiXIE
+      - FLACME
+      - FrB
+      - HOUND
+      - JRO
+      - KINDA
+      - MAHOU
+      - MFDOS
+      - MLS
+      - Mrflac
+      - MUNDANE
+      - MyDad
+      - OBZEN
+      - OTT
+      - PERFECT
+      - PLAYLiST
+      - PTC
+      - PWT
+      - RAGEFLAC
+      - RAWBEATS
+      - RECTiFY
+      - ROSiN
+      - STAX
+      - THEVOiD
+      - TiMES
+      - TM
+      - TVRf
+      - VEXED
+      - WAVED
+      - YARD
--- a/config/scope/rutracker.yaml
+++ b/config/scope/rutracker.yaml
--- a/config/scope/web.yaml
+++ b/config/scope/web.yaml
@ -0,0 +1,3 @@
+role: web
+http:
+  port: 19576
--- a/config/secret.yaml-sample
+++ b/config/secret.yaml-sample
@ -0,0 +1,6 @@
+db:
+  driver: mysql
+  data_source_name: <user:pass@(localhost:3306)/bd_name>
+jwt:
+  ttl: 6h12m24s
+  sign_key: <your key>
--- a/go.mod
+++ b/go.mod
@ -5,30 +5,46 @@ go 1.23.1
 require (
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-chi/cors v1.2.1
+	github.com/go-chi/jwtauth/v5 v5.3.1
+	github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
 	github.com/go-sql-driver/mysql v1.8.1
+	github.com/golang-module/carbon/v2 v2.3.12
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.19.0
+	golang.org/x/net v0.29.0
 )

 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
+	github.com/lestrrat-go/httpcc v1.0.1 // indirect
+	github.com/lestrrat-go/httprc v1.0.4 // indirect
+	github.com/lestrrat-go/iter v1.0.2 // indirect
+	github.com/lestrrat-go/jwx/v2 v2.0.20 // indirect
+	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -1,9 +1,13 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@ -12,8 +16,18 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
 github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
 github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-chi/jwtauth/v5 v5.3.1 h1:1ePWrjVctvp1tyBq5b/2ER8Th/+RbYc7x4qNsc5rh5A=
+github.com/go-chi/jwtauth/v5 v5.3.1/go.mod h1:6Fl2RRmWXs3tJYE1IQGX81FsPoGqDwq9c15j52R5q80=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
+github.com/golang-module/carbon/v2 v2.3.12 h1:VC1DwN1kBwJkh5MjXmTFryjs5g4CWyoM8HAHffZPX/k=
+github.com/golang-module/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@ -24,6 +38,18 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
+github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
+github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
+github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
+github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8=
+github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
+github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
+github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
+github.com/lestrrat-go/jwx/v2 v2.0.20 h1:sAgXuWS/t8ykxS9Bi2Qtn5Qhpakw1wrcjxChudjolCc=
+github.com/lestrrat-go/jwx/v2 v2.0.20/go.mod h1:UlCSmKqw+agm5BsOBfEAbTvKsEApaGNqHAEUTv5PJC4=
+github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
+github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
@ -43,6 +69,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
@ -58,6 +86,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
@ -65,16 +94,54 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
 go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
--- a/helper/parser/html.go
+++ b/helper/parser/html.go
@ -0,0 +1,63 @@
+package parser
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+func setClient(url string) (*http.Response, error) {
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
+
+	return client.Do(req)
+}
+
+func HTMLSourceFromURL(url string) (*html.Node, error) {
+	resp, err := setClient(url)
+	defer func(Body io.ReadCloser) {
+		fmt.Printf("%v\n", Body == nil)
+
+		err = Body.Close()
+		if err != nil {
+			slog.Error("closing response body", "err", err)
+		}
+	}(resp.Body)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= 400 {
+		return nil, errors.New("http status code: " + strconv.Itoa(resp.StatusCode))
+	}
+
+	doc, err := html.Parse(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return doc, nil
+}
+
+func HTMLSource(htmlSource string) (*html.Node, error) {
+	doc, err := html.Parse(strings.NewReader(htmlSource))
+	if err != nil {
+		return nil, err
+	}
+	return doc, nil
+}
+
+//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
--- a/internal/bootstrap.go
+++ b/internal/bootstrap.go
@ -5,16 +5,15 @@ import (

 	"github.com/spf13/viper"

-	"git.amok.space/yevhen/resource-scraper/internal/config"
 	"git.amok.space/yevhen/resource-scraper/internal/db"
 	"git.amok.space/yevhen/resource-scraper/pkg/handler"
 	"git.amok.space/yevhen/resource-scraper/pkg/repository"
 	"git.amok.space/yevhen/resource-scraper/pkg/service"
-	"git.amok.space/yevhen/resource-scraper/types"
+	"git.amok.space/yevhen/resource-scraper/types/constant"
 )

 func Bootstrap() {
-	config.New()
+	//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang

 	dbase := db.New()
 	repos := repository.New(dbase)
@ -22,10 +21,12 @@ func Bootstrap() {
 	handlers := handler.New(services)

 	switch viper.GetString("role") {
-	case types.RoleConsole:
+	case constant.RoleConsole:
 		fmt.Printf("init console console: %s\n", handlers.InitConsole())
 		break
+	case constant.RoleWeb:
+		fmt.Printf("who: %s\n", handlers.InitRoutes())
+		///http.Run()
+		break
 	}
-
-	///http.Run()
 }
--- a/internal/config/config.go
+++ b/internal/config/config.go
@ -3,32 +3,62 @@ package config
 import (
 	"fmt"
 	"log"
+	"log/slog"
 	"path/filepath"
+	"slices"

 	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/types/constant"
 )

 func New() {
-	configFilePath := viper.GetString("config-file")
-	configDir := "./" + filepath.Dir(configFilePath)
+	configFilePath := viper.GetString(constant.FlagConfigFile)
+	configName := filepath.Base(configFilePath)
+
+	dir := filepath.Dir(configFilePath)
+	if dir == configName && dir == "." {
+		configName = "default"
+	}
+
+	configDir, _ := filepath.Abs(dir)

-	viper.SetConfigName(filepath.Base(configFilePath))
 	viper.AddConfigPath(configDir)
-
-	err := viper.ReadInConfig() // Find and read the config file
-	if err != nil {             // Handle errors reading the config file
+	viper.SetConfigName(configName)
+	err := viper.ReadInConfig()
+	if err != nil { // Handle errors reading the config file
 		panic(fmt.Errorf("fatal error config file: %w", err))
 	}

-	viper.SetDefault("ConfigDir", configDir)
-	scope := viper.GetString("scope")
-	if scope != "" {
-		viper.SetConfigName(scope)
-		viper.AddConfigPath(configDir)
-		err := viper.MergeInConfig()
-		if err != nil {
-			log.Fatalf("fatal error config file: %v", err)
-			return
+	viper.SetConfigName("secret")
+	err = viper.MergeInConfig()
+	if err != nil { // Handle errors reading the config file
+		panic(fmt.Errorf("fatal error secret file: %w", err))
+	}
+
+	slog.Info("using config", "path", filepath.Join(configDir, configName))
+	viper.SetDefault(constant.CfgKeyConfigDir, configDir)
+
+	// Scopes validating
+	scope := viper.GetString(constant.CfgKeyScopeEnable)
+	scopesAllowed := viper.GetStringSlice("scope.allow")
+
+	if !slices.Contains(scopesAllowed, scope) {
+		scope = viper.GetString("scope.default")
+
+		if scope == "" {
+			scope = constant.ScopeInfo
 		}
 	}
+
+	fmt.Println("scope", scope)
+
+	viper.SetConfigName(scope)
+	viper.AddConfigPath(filepath.Join(configDir, "scope"))
+	err = viper.MergeInConfig()
+	if err == nil {
+		viper.SetDefault(constant.CfgKeyScopeEnable, scope)
+	} else {
+		log.Fatalf("fatal error config file: %v", err)
+	}
 }
--- a/internal/flag.go
+++ b/internal/flag.go
@ -8,6 +8,8 @@ import (

 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/types/constant"
 )

 const usage = `Music Database (MDB) server/cli craftware'
@ -25,19 +27,18 @@ The Go runtime version: %s
 Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`

 const (
-	maxPasswordLength = 20
 	version           = "0.1"
-	defaultConfigPath = "./config/default"
+	defaultConfigPath = "config/default"
 )

 func ParseFlags() {
-	flag.Bool("h", false, "")
-	flag.Bool("help", false, "")
-	flag.Bool("v", false, "")
-	flag.Bool("version", false, "")
-	flag.Bool("debug", false, "")
-	flag.String("config-file", defaultConfigPath, "config file location used for the program")
-	flag.String("scope", "", "")
+	flag.Bool(constant.FlagHelpShort, false, "")
+	flag.Bool(constant.FlagHelp, false, "")
+	flag.Bool(constant.FlagVersionShort, false, "")
+	flag.Bool(constant.FlagVersion, false, "")
+	flag.Bool(constant.FlagDebug, false, "")
+	flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location used for the program")
+	flag.String(constant.FlagScopeEnable, "", "")

 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	pflag.Parse()
@ -46,10 +47,10 @@ func ParseFlags() {
 		log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
 	}

-	if viper.GetBool("h") || viper.GetBool("help") {
+	if viper.GetBool(constant.FlagHelpShort) || viper.GetBool(constant.FlagHelp) {
 		msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
 		fmt.Println(msg)
-	} else if viper.GetBool("v") || viper.GetBool("version") {
+	} else if viper.GetBool(constant.FlagVersionShort) || viper.GetBool(constant.FlagVersion) {
 		fmt.Println("MDB version", version)
 	}
 }
--- a/59
+++ b/59
@ -0,0 +1,59 @@
+scope prescene
+FLOAT: 33.000000, 12.000000, 3
+false
+======================  1  ==============================
+{Id:825647 Type:prescene TypeId:4313240 Title:Phaethon-Wielder_Of_The_Steel-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/phaethon-wielder_of_the_steel-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:09:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  2  ==============================
+{Id:825648 Type:prescene TypeId:4313238 Title:Helvetets_Port-Warlords-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/helvetets_port-warlords-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:08:33 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  4  ==============================
+{Id:825649 Type:prescene TypeId:4313236 Title:Circle_of_Ouroborus-Ajattoman_Uni-FI-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/circle_of_ouroborus-ajattoman_uni-fi-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:07:20 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  5  ==============================
+{Id:825650 Type:prescene TypeId:4313234 Title:Blitzkrieg-Blitzkrieg-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/blitzkrieg-blitzkrieg-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:06:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  6  ==============================
+{Id:825651 Type:prescene TypeId:4313233 Title:Antioch-Onward_With_Obliteration-EP-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/antioch-onward_with_obliteration-ep-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:05:27 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  8  ==============================
+{Id:825631 Type:prescene TypeId:4313221 Title:Wretchgod-Suffering_Upon_Suffering-EP-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-suffering_upon_suffering-ep-24bit-web-flac-2024-moonblood/ Created:2024-09-10 14:07:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  9  ==============================
+{Id:825632 Type:prescene TypeId:4313220 Title:Wretchgod-Stygian_Blood_Ritual-EP-16BIT-WEB-FLAC-2022-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-stygian_blood_ritual-ep-16bit-web-flac-2022-moonblood/ Created:2024-09-10 14:06:51 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  11  ==============================
+{Id:825633 Type:prescene TypeId:4313188 Title:1349-The_Infernal_Pathway-REPACK-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-repack-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:34:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  15  ==============================
+{Id:825634 Type:prescene TypeId:4313177 Title:1349-The_Infernal_Pathway-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:17:42 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  16  ==============================
+{Id:825635 Type:prescene TypeId:4313175 Title:1349-Revelations_of_the_Black_Flame-16BIT-WEB-FLAC-2009-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-revelations_of_the_black_flame-16bit-web-flac-2009-moonblood/ Created:2024-09-10 13:16:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  17  ==============================
+{Id:825636 Type:prescene TypeId:4313173 Title:1349-Massive_Cauldron_of_Chaos-16BIT-WEB-FLAC-2014-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-massive_cauldron_of_chaos-16bit-web-flac-2014-moonblood/ Created:2024-09-10 13:15:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  19  ==============================
+{Id:825637 Type:prescene TypeId:4313171 Title:1349-Liberation-16BIT-WEB-FLAC-2003-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-liberation-16bit-web-flac-2003-moonblood/ Created:2024-09-10 13:14:38 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  21  ==============================
+{Id:825638 Type:prescene TypeId:4313168 Title:1349-Hellfire-16BIT-WEB-FLAC-2005-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-hellfire-16bit-web-flac-2005-moonblood/ Created:2024-09-10 13:13:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  22  ==============================
+{Id:825639 Type:prescene TypeId:4313166 Title:1349-Demonoir-REISSUE-16BIT-WEB-FLAC-2011-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-demonoir-reissue-16bit-web-flac-2011-moonblood/ Created:2024-09-10 13:12:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  23  ==============================
+{Id:825640 Type:prescene TypeId:4313164 Title:1349-Beyond_the_Apocalypse-16BIT-WEB-FLAC-2004-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-beyond_the_apocalypse-16bit-web-flac-2004-moonblood/ Created:2024-09-10 13:12:00 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+Sleeping... 0
+false
+======================  1  ==============================
+{Id:825652 Type:prescene TypeId:4300530 Title:Zuckerbaby--Zuckerbaby-CD-FLAC-1997-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/zuckerbaby-zuckerbaby-cd-flac-1997-meowzik/ Created:2024-08-31 18:17:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  2  ==============================
+{Id:825653 Type:prescene TypeId:4300529 Title:Silmarils--Vegas_76-FR-CD-FLAC-2000-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/silmarils-vegas_76-fr-cd-flac-2000-meowzik/ Created:2024-08-31 18:16:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  3  ==============================
+{Id:825654 Type:prescene TypeId:4300527 Title:Michael_Kaeshammer--Days_Like_These-CD-FLAC-2007-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/michael_kaeshammer-days_like_these-cd-flac-2007-meowzik/ Created:2024-08-31 18:14:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  4  ==============================
+{Id:825655 Type:prescene TypeId:4300521 Title:Butch_Walker--Letters-CD-FLAC-2004-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/butch_walker-letters-cd-flac-2004-meowzik/ Created:2024-08-31 18:09:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  16  ==============================
+{Id:825656 Type:prescene TypeId:4300320 Title:Embodiment_Elimination-Metamorphosis_Incarnate_Through_Genetic_Devastation-(ISR071-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/embodiment_elimination-metamorphosis_incarnate_through_genetic_devastation-isr071-24-cd-flac-2024-86d/ Created:2024-08-31 11:19:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  19  ==============================
+{Id:825657 Type:prescene TypeId:4300309 Title:Putrified_Degradation-Manufractured-(ISR072-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/putrified_degradation-manufractured-isr072-24-cd-flac-2024-86d/ Created:2024-08-31 10:45:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  20  ==============================
+{Id:825658 Type:prescene TypeId:4300307 Title:Post_Mortal_Possession-The_Dead_Space_Between_the_Stars-(LORD091)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/post_mortal_possession-the_dead_space_between_the_stars-lord091-cd-flac-2024-86d/ Created:2024-08-31 10:43:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  21  ==============================
+{Id:825659 Type:prescene TypeId:4300306 Title:Invirulant-Necrothuggin-(ISR070-24)-CDEP-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/invirulant-necrothuggin-isr070-24-cdep-flac-2024-86d/ Created:2024-08-31 10:41:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  22  ==============================
+{Id:825660 Type:prescene TypeId:4300304 Title:Dead_and_Dripping-Disillusioned_by_Excessive_Human_Consumption-(LORD092)-DEMO_REISSUE-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/dead_and_dripping-disillusioned_by_excessive_human_consumption-lord092-demo_reissue-cd-flac-2024-86d/ Created:2024-08-31 10:38:13 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+Sleeping... 1
+scope: prescene
+[1 45]
+init console console: ps
+
+Execution took 12.6276275s
--- a/pkg/handler/handler.go
+++ b/pkg/handler/handler.go
@ -1,7 +1,10 @@
 package handler

 import (
+	"github.com/spf13/viper"
+
 	"git.amok.space/yevhen/resource-scraper/pkg/service"
+	"git.amok.space/yevhen/resource-scraper/types/constant"
 )

 type Handler struct {
@ -13,16 +16,17 @@ func New(services *service.Service) *Handler {
 }

 func (h *Handler) InitConsole() string {
-	return h.rutracker()
+	switch viper.GetString(constant.CfgKeyScopeEnable) {
+	case constant.ScopeRuTracker:
+		return h.rutracker()
+	case constant.ScopePrescene:
+		return h.prescene()
+	}
+
+	return "no scope chosen"
 }

-//func (h *Handler) Base(services *service.Service) *Handler {
-//	return &Handler{services: services}
-//}
-
-//func (h *Handler) InitApi() *chi.Mux {
-//	api := chi.NewRouter()
-//	api.Get("/", web.ApiFallbackHandler)
-//
-//	return api
-//}
+func (h *Handler) InitRoutes() string {
+	//TODO:
+	return "i am the web initiator"
+}
--- a/pkg/handler/prescene.go
+++ b/pkg/handler/prescene.go
@ -0,0 +1,46 @@
+package handler
+
+import (
+	"fmt"
+	"log/slog"
+	"math"
+	"strconv"
+	"time"
+
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/types/constant"
+)
+
+func (h *Handler) prescene() string {
+	pagesToScrape := []string{"1"}
+	levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
+	if levels > 1 {
+		var q = (60.0 / 100.0) * (100.0 / viper.GetFloat64(constant.CfgKeyLevelsToScrape))
+		minute := float64(time.Now().Minute())
+		if minute < 1 {
+			minute = 1
+		}
+
+		fmt.Printf("FLOAT: %f, %f, %v\n", minute, q, math.Ceil(minute/q))
+
+		hour := time.Now().Hour()
+		if hour == 1 {
+			hour = 25
+		} else if hour == 0 {
+			hour = 1
+		}
+
+		page := hour * int(math.Ceil(minute/q))
+		pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
+	}
+
+	_, err := h.services.Prescene.GetPage(pagesToScrape)
+	if err != nil {
+		slog.Error("error occurred while getting page: ", "err", err.Error())
+	}
+
+	fmt.Printf("%+v\n", pagesToScrape)
+
+	return "ps"
+}
--- a/pkg/handler/rutracker.go
+++ b/pkg/handler/rutracker.go
@ -12,10 +12,12 @@ func (h *Handler) rutracker() string {
 	key := fmt.Sprintf("topic.%v", time.Now().Hour())
 	topics := viper.GetStringSlice(key)

-	err := h.services.Rutracker.GetTopic(topics)
+	_, err := h.services.Rutracker.GetTopic(topics)
 	if err != nil {
-		slog.Error("error occurred while getting topic: ", err.Error())
+		slog.Error("error occurred while getting topic: ", "err", err.Error())
 	}

+	//fmt.Printf("%+v", rt)
+
 	return "rt"
 }
--- a/pkg/repository/prescene.go
+++ b/pkg/repository/prescene.go
@ -0,0 +1,108 @@
+package repository
+
+import (
+	"fmt"
+	"log/slog"
+	"regexp"
+	"slices"
+	"strconv"
+	"time"
+
+	"github.com/go-shiori/dom"
+	"github.com/golang-module/carbon/v2"
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/helper/parser"
+	"git.amok.space/yevhen/resource-scraper/pkg/repository/table"
+	"git.amok.space/yevhen/resource-scraper/types/constant"
+	"git.amok.space/yevhen/resource-scraper/types/model"
+)
+
+type Prescene struct {
+	db *sqlx.DB
+}
+
+func NewPresceneRepository(db *sqlx.DB) *Prescene {
+	return &Prescene{db: db}
+}
+
+func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
+	entries := make([]model.ExternalSources, 0)
+	endpoint := viper.GetString(constant.CfgKeyEndpoint)
+	//scope := viper.GetString(constant.CfgKeyScopeEnable)
+	tags := viper.GetStringMapStringSlice("groups.tags")
+
+	for _, t := range pageNumbers {
+		if t != "1" {
+			endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
+		}
+		doc, err := parser.HTMLSourceFromURL(endpoint)
+		//doc, err := parser.HTMLSourceFromURL("https://mdb.amok.space/$/scnlog.html")
+
+		if err != nil {
+			slog.Error("Parse error", "err", err)
+			continue
+		}
+
+		if doc == nil {
+			slog.Warn("Document is nil", "err", err)
+			continue
+		}
+
+		for _, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
+			var es model.ExternalSources
+			columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
+
+			title := dom.QuerySelector(item, ".title")
+			if title != nil {
+				anchor := dom.QuerySelector(title, "h1 > a")
+				if anchor != nil {
+					es.Type = constant.ScopePrescene
+					es.Title = dom.GetAttribute(anchor, "title")
+					es.ExSource = dom.GetAttribute(anchor, "href")
+					pattern := regexp.MustCompile(`(?is)-(\w+)$`)
+					es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
+
+					for flag, groups := range tags {
+						if slices.Contains(groups, es.Releaser) {
+							es.A = flag
+							es.H = flag
+							columns = append(columns, "a", "h")
+							break
+						}
+					}
+				}
+
+				if es.A == constant.TagIgnore {
+					slog.Info("Skipped", "releaser", es.Releaser)
+					continue
+				}
+
+				localtime := dom.QuerySelector(title, "small > span.localtime")
+				if localtime != nil {
+					lc := dom.GetAttribute(localtime, "data-lttime")
+					es.Created = carbon.Parse(lc)
+				}
+			}
+
+			cls := dom.GetAttribute(item, "class")
+			pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
+			es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
+
+			//fmt.Println("====================== ", i, " ==============================")
+			esModel := table.ExternalSources{Columns: columns}
+			entry := esModel.InsertOnDuplicate(es, s.db)
+			entries = append(entries, entry)
+
+			//fmt.Printf("%+v\n", entry)
+		}
+
+		//fmt.Println("Sleeping...", j)
+		time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
+	}
+
+	//fmt.Printf("scope: %v\n", scope)
+
+	return entries, nil
+}
--- a/pkg/repository/repository.go
+++ b/pkg/repository/repository.go
@ -2,22 +2,18 @@ package repository

 import (
 	"github.com/jmoiron/sqlx"
-	"github.com/spf13/viper"

-	"git.amok.space/yevhen/resource-scraper/types"
+	"git.amok.space/yevhen/resource-scraper/types/interface"
 )

 type Repository struct {
-	types.Rutracker
+	_interface.Rutracker
+	_interface.Prescene
 }

 func New(db *sqlx.DB) *Repository {
-	switch viper.GetString("scope") {
-	case types.RuTracker:
-		return &Repository{
-			Rutracker: NewRutracker(db),
-		}
+	return &Repository{
+		Rutracker: NewRutrackerRepository(db),
+		Prescene:  NewPresceneRepository(db),
 	}
-
-	return &Repository{}
 }
--- a/pkg/repository/rutracker.go
+++ b/pkg/repository/rutracker.go
@ -8,60 +8,59 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
-	"time"

+	"github.com/golang-module/carbon/v2"
 	"github.com/jmoiron/sqlx"
 	"github.com/spf13/viper"

-	iface "git.amok.space/yevhen/resource-scraper/types"
-	"git.amok.space/yevhen/resource-scraper/types/table"
+	"git.amok.space/yevhen/resource-scraper/pkg/repository/table"
+	"git.amok.space/yevhen/resource-scraper/types/constant"
+	"git.amok.space/yevhen/resource-scraper/types/model"
+	"git.amok.space/yevhen/resource-scraper/types/resource"
 )

 type Rutracker struct {
 	db *sqlx.DB
 }

-func NewRutracker(db *sqlx.DB) *Rutracker {
+func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
 	return &Rutracker{db: db}
 }

-func (s *Rutracker) GetTopic(topics []string) error {
-	endpoint := viper.GetString("endpoint")
+func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
+	endpoint := viper.GetString(constant.CfgKeyEndpoint)
+	entries := make([]model.ExternalSources, 0)
+	columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}

 	for _, t := range topics {
 		topic, err := fetch(fmt.Sprintf(endpoint, t))
 		if err != nil {
 			slog.Error("couldn't parse topic data", "err", err.Error())
+			return entries, err
 		}

-		for i, e := range topic.Entry {
-			var id int
-			var es table.ExternalSources
-
+		for _, e := range topic.Entry {
+			var es model.ExternalSources
 			u, _ := url.Parse(e.Link.Href)
-			es.Type = "rutracker"
+
+			es.Type = constant.ScopeRuTracker
 			es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
 			es.Title = e.Title
 			es.TypeSubsectionId, _ = strconv.Atoi(t)
 			es.Releaser = e.Author.Name
-			es.Created, _ = time.Parse(time.RFC3339, e.Updated)
-			created := es.Created.Format(iface.DateTimeFormat)
+			es.Created = carbon.Parse(e.Updated)

-			query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable)
-
-			row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created)
-			if err = row.Scan(&id); err != nil {
-				return err
-			}
-
-			fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>")
+			esModel := table.ExternalSources{Columns: columns}
+			entry := esModel.InsertOnDuplicate(es, s.db)
+			entries = append(entries, entry)
+			//fmt.Printf("%+v\n\n\n", entry)
 		}
 	}

-	return nil
+	return entries, nil
 }

-func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
+func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
 	resp, err := http.Get(endpoint)
 	if err != nil {
 		slog.Error("couldn't fetch data", endpoint, err.Error())
@ -75,7 +74,7 @@ func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
 		}
 	}(resp.Body)

-	topic := &iface.RutrackerAtomTopic{}
+	topic := &resource.RutrackerAtomTopic{}

 	if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
 		return nil, err
--- a/pkg/repository/table/external_sources.go
+++ b/pkg/repository/table/external_sources.go
@ -0,0 +1,34 @@
+package table
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/jmoiron/sqlx"
+
+	"git.amok.space/yevhen/resource-scraper/types/constant"
+	"git.amok.space/yevhen/resource-scraper/types/model"
+)
+
+type ExternalSources struct {
+	Columns []string
+}
+
+func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.DB) model.ExternalSources {
+	stmt := "INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE title=:title, created=:created RETURNING id"
+	placeholders := strings.Join(f.Columns, ", :")
+	placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
+	query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
+
+	//fmt.Printf("%s\n", query)
+
+	if rows, err := db.NamedQuery(query, &es); err == nil {
+		for rows.Next() {
+			es.Error = rows.StructScan(&es)
+		}
+	} else {
+		es.Error = err
+	}
+
+	return es
+}
--- a/pkg/service/prescene.go
+++ b/pkg/service/prescene.go
@ -0,0 +1,18 @@
+package service
+
+import (
+	"git.amok.space/yevhen/resource-scraper/types/interface"
+	"git.amok.space/yevhen/resource-scraper/types/model"
+)
+
+type PresceneService struct {
+	repo _interface.Prescene
+}
+
+func NewPresceneService(repo _interface.Prescene) *PresceneService {
+	return &PresceneService{repo: repo}
+}
+
+func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
+	return p.repo.GetPage(pageNumbers)
+}
--- a/pkg/service/rutracker.go
+++ b/pkg/service/rutracker.go
@ -1,17 +1,18 @@
 package service

 import (
-	iface "git.amok.space/yevhen/resource-scraper/types"
+	"git.amok.space/yevhen/resource-scraper/types/interface"
+	"git.amok.space/yevhen/resource-scraper/types/model"
 )

 type RutrackerService struct {
-	repo iface.Rutracker
+	repo _interface.Rutracker
 }

-func NewRutrackerService(repo iface.Rutracker) *RutrackerService {
+func NewRutrackerService(repo _interface.Rutracker) *RutrackerService {
 	return &RutrackerService{repo: repo}
 }

-func (s *RutrackerService) GetTopic(topic []string) error {
+func (s *RutrackerService) GetTopic(topic []string) ([]model.ExternalSources, error) {
 	return s.repo.GetTopic(topic)
 }
--- a/pkg/service/service.go
+++ b/pkg/service/service.go
@ -1,24 +1,18 @@
 package service

 import (
-	"github.com/spf13/viper"
-
 	"git.amok.space/yevhen/resource-scraper/pkg/repository"
-	"git.amok.space/yevhen/resource-scraper/types"
+	"git.amok.space/yevhen/resource-scraper/types/interface"
 )

 type Service struct {
-	types.Rutracker
+	_interface.Rutracker
+	_interface.Prescene
 }

 func New(repos *repository.Repository) *Service {
-
-	switch viper.GetString("scope") {
-	case types.RuTracker:
-		return &Service{
-			Rutracker: NewRutrackerService(repos.Rutracker),
-		}
+	return &Service{
+		Rutracker: NewRutrackerService(repos.Rutracker),
+		Prescene:  NewPresceneService(repos.Prescene),
 	}
-
-	return &Service{}
 }
--- a/types/constant/config.go
+++ b/types/constant/config.go
@ -0,0 +1,10 @@
+package constant
+
+const (
+	CfgKeyScopeEnable              = FlagScopeEnable
+	CfgKeyConfigDir                = "config-dir"
+	CfgKeySleepBeforeNextIteration = "sleep-before-next-iteration"
+	CfgKeyEndpoint                 = "endpoint"
+	CfgKeyEndpointNext             = "endpoint-next"
+	CfgKeyLevelsToScrape           = "levels-to-scrape"
+)
--- a/types/constant/constants.go
+++ b/types/constant/constants.go
@ -0,0 +1,18 @@
+package constant
+
+const (
+	ScopeRuTracker string = "rutracker"
+	ScopePrescene  string = "prescene"
+	ScopeWeb       string = "web"
+	ScopeInfo      string = "info"
+)
+
+const (
+	RoleConsole string = "console"
+	RoleWeb     string = "web"
+)
+
+const (
+	TagIgnore            string = "ignore"
+	ExternalSourcesTable string = "external_sources"
+)
--- a/types/constant/flag.go
+++ b/types/constant/flag.go
@ -0,0 +1,11 @@
+package constant
+
+const (
+	FlagHelp         = "help"
+	FlagHelpShort    = "h"
+	FlagDebug        = "debug"
+	FlagVersion      = "version"
+	FlagVersionShort = "v"
+	FlagConfigFile   = "config-file"
+	FlagScopeEnable  = "scope-enable"
+)
--- a/types/constants.go
+++ b/types/constants.go
@ -1,14 +0,0 @@
-package types
-
-const (
-	RuTracker string = "rutracker"
-)
-
-const (
-	RoleConsole string = "console"
-)
-
-const (
-	DateTimeFormat       string = "2006-01-02 15:04:05"
-	ExternalSourcesTable string = "external_sources"
-)
--- a/types/interface/interfaces.go
+++ b/types/interface/interfaces.go
@ -0,0 +1,11 @@
+package _interface
+
+import "git.amok.space/yevhen/resource-scraper/types/model"
+
+type Rutracker interface {
+	GetTopic(topics []string) ([]model.ExternalSources, error)
+}
+
+type Prescene interface {
+	GetPage(pageNumbers []string) ([]model.ExternalSources, error)
+}
--- a/types/interfaces.go
+++ b/types/interfaces.go
@ -1,5 +0,0 @@
-package types
-
-type Rutracker interface {
-	GetTopic(topics []string) error
-}
--- a/types/model/external_sources.go
+++ b/types/model/external_sources.go
@ -0,0 +1,42 @@
+package model
+
+import (
+	"github.com/golang-module/carbon/v2"
+)
+
+type Type string
+
+/*const (
+	mmt Type = iota
+	prescene
+	rutracker
+	locate
+	darkabyss
+	bfm
+	trash
+	nnmc
+	stb
+	bandcamp
+	deathgrind
+)*/
+
+type ExternalSources struct {
+	Id                 int           `json:"id" db:"id"`
+	Type               string        `json:"type" db:"type"`
+	TypeId             int           `json:"type_id" db:"type_id"`
+	Title              string        `json:"title" db:"title"`
+	TypeSubsectionId   int           `json:"type_subsection_id" db:"type_subsection_id"`
+	Releaser           string        `json:"releaser" db:"releaser"`
+	ExSource           string        `json:"ex_source" db:"eXsource"`
+	Created            carbon.Carbon `json:"created_at" db:"created"`
+	Modified           carbon.Carbon `json:"modified_at" db:"modified"`
+	Contents           string        `json:"contents" db:"contents"`
+	A                  string        `json:"a" db:"a"`
+	H                  string        `json:"h" db:"h"`
+	Fingerprint        string        `json:"fingerprint" db:"fingerprint"`
+	FsFingerprint      string        `json:"fs_fingerprint" db:"fs_fingerprint"`
+	Vid                int           `json:"vid" db:"vid"`
+	G                  string        `json:"g" db:"g"`
+	ExternalSourcesCol string        `json:"external_sources_column" db:"external_sourcescol"`
+	Error              error
+}
--- a/types/resource/rutracker.go
+++ b/types/resource/rutracker.go
@ -1,4 +1,4 @@
-package types
+package resource

 import "encoding/xml"

--- a/types/resource/scnlog.go
+++ b/types/resource/scnlog.go
@ -0,0 +1,46 @@
+package resource
+
+import "encoding/xml"
+
+type ScnLogRss struct {
+	XMLName xml.Name `xml:"rss"`
+	Text    string   `xml:",chardata"`
+	Version string   `xml:"version,attr"`
+	Content string   `xml:"content,attr"`
+	Wfw     string   `xml:"wfw,attr"`
+	Dc      string   `xml:"dc,attr"`
+	Atom    string   `xml:"atom,attr"`
+	Sy      string   `xml:"sy,attr"`
+	Slash   string   `xml:"slash,attr"`
+	Channel struct {
+		Text  string `xml:",chardata"`
+		Title string `xml:"title"`
+		Link  struct {
+			Text string `xml:",chardata"`
+			Href string `xml:"href,attr"`
+			Rel  string `xml:"rel,attr"`
+			Type string `xml:"type,attr"`
+		} `xml:"link"`
+		Description     string `xml:"description"`
+		LastBuildDate   string `xml:"lastBuildDate"`
+		Language        string `xml:"language"`
+		UpdatePeriod    string `xml:"updatePeriod"`
+		UpdateFrequency string `xml:"updateFrequency"`
+		Generator       string `xml:"generator"`
+		Item            []struct {
+			Text     string   `xml:",chardata"`
+			Title    string   `xml:"title"`
+			Link     string   `xml:"link"`
+			Comments string   `xml:"comments"`
+			Creator  string   `xml:"creator"`
+			PubDate  string   `xml:"pubDate"`
+			Category []string `xml:"category"`
+			Guid     struct {
+				Text        string `xml:",chardata"`
+				IsPermaLink string `xml:"isPermaLink,attr"`
+			} `xml:"guid"`
+			Description string `xml:"description"`
+			CommentRss  string `xml:"commentRss"`
+		} `xml:"item"`
+	} `xml:"channel"`
+}
--- a/types/table/external_sources.go
+++ b/types/table/external_sources.go
@ -1,39 +0,0 @@
-package table
-
-import "time"
-
-type Type string
-
-/*const (
-	mmt Type = iota
-	prescene
-	rutracker
-	locate
-	darkabyss
-	bfm
-	trash
-	nnmc
-	stb
-	bandcamp
-	deathgrind
-)*/
-
-type ExternalSources struct {
-	Id                 int       `json:"id" db:"id"`
-	Type               string    `json:"type" db:"type"`
-	TypeId             int       `json:"type_id" db:"type_id"`
-	Title              string    `json:"title" db:"title"`
-	TypeSubsectionId   int       `json:"type_subsection_id" db:"type_subsection_id"`
-	Releaser           string    `json:"releaser" db:"releaser"`
-	ExSource           string    `json:"ex_source" db:"eXsource"`
-	Created            time.Time `json:"created_at" db:"created"`
-	Modified           time.Time `json:"modified_at" db:"modified"`
-	Contents           string    `json:"contents" db:"contents"`
-	A                  string    `json:"a" db:"a"`
-	H                  string    `json:"h" db:"h"`
-	Fingerprint        string    `json:"fingerprint" db:"fingerprint"`
-	FsFingerprint      string    `json:"fs_fingerprint" db:"fs_fingerprint"`
-	Vid                int       `json:"vid" db:"vid"`
-	G                  string    `json:"g" db:"g"`
-	ExternalSourcesCol string    `json:"external_sources_column" db:"external_sourcescol"`
-}