upadted to 1.24.0; fixed stb and spiddped doubles in precene
This commit is contained in:
parent
0ecf0ddec1
commit
203f834f65
@ -1,6 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="GoLinterSettings">
|
||||
<option name="concurrency" value="5" />
|
||||
<option name="enabledLinters">
|
||||
<list>
|
||||
<option value="ineffassign" />
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<profile version="1.0">
|
||||
<option name="myName" value="Project Default" />
|
||||
<inspection_tool class="GoDfaErrorMayBeNotNil" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<functions>
|
||||
<function importPath="net/http" name="Get" />
|
||||
</functions>
|
||||
</inspection_tool>
|
||||
</profile>
|
||||
</component>
|
||||
@ -8,14 +8,18 @@ import (
|
||||
|
||||
scraper "git.amok.space/yevhen/resource-scraper/internal"
|
||||
"git.amok.space/yevhen/resource-scraper/internal/config"
|
||||
"git.amok.space/yevhen/resource-scraper/internal/logging"
|
||||
)
|
||||
|
||||
var st time.Time
|
||||
var (
|
||||
st time.Time
|
||||
)
|
||||
|
||||
//var tokenAuth *jwtauth.JWTAuth
|
||||
|
||||
func init() {
|
||||
st = time.Now()
|
||||
logging.New()
|
||||
scraper.ParseFlags()
|
||||
config.New()
|
||||
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
scope:
|
||||
allow: [info, web, rutracker, prescene]
|
||||
default: web
|
||||
allow: [info, web, rutracker, prescene, stb, metal-archives, console]
|
||||
default: console
|
||||
mail:
|
||||
amok.space:
|
||||
dial-tls: mail.amok.space:993
|
||||
4
config/scope/console.yaml
Normal file
4
config/scope/console.yaml
Normal file
@ -0,0 +1,4 @@
|
||||
role: console
|
||||
console:
|
||||
cmd:
|
||||
create: [scope]
|
||||
2
config/scope/metal-archives.yaml
Normal file
2
config/scope/metal-archives.yaml
Normal file
@ -0,0 +1,2 @@
|
||||
role: console
|
||||
metal-archives:
|
||||
@ -3,11 +3,12 @@ endpoint: https://scnlog.me/music/flac/
|
||||
endpoint-next: page/%s/
|
||||
sleep-before-next-iteration: 5s
|
||||
levels-to-scrape: 5
|
||||
cmd: go run .\cmd\main.go --scope-enable prescene --single-uri page/1/?s=<%search_criteria%>
|
||||
groups:
|
||||
tags:
|
||||
neformat:
|
||||
- SHGZ
|
||||
remove:
|
||||
ignore:
|
||||
- BIGLOVE
|
||||
- CT
|
||||
- ENViED
|
||||
@ -16,7 +17,6 @@ groups:
|
||||
- PTCx
|
||||
- SHGZ
|
||||
- XiVERO
|
||||
ignore:
|
||||
- "401"
|
||||
- 6DM
|
||||
- AFO
|
||||
@ -44,12 +44,14 @@ groups:
|
||||
- JRO
|
||||
- KINDA
|
||||
- MAHOU
|
||||
- MEOWZiK
|
||||
- MARiBOR
|
||||
- MFDOS
|
||||
- MLS
|
||||
- Mrflac
|
||||
- MUNDANE
|
||||
- MyDad
|
||||
- OBZEN
|
||||
# - OBZEN
|
||||
- OTT
|
||||
- PERFECT
|
||||
- PLAYLiST
|
||||
@ -61,9 +63,15 @@ groups:
|
||||
- ROSiN
|
||||
- STAX
|
||||
- THEVOiD
|
||||
- TiMES
|
||||
# - TiMES
|
||||
- TM
|
||||
- TVRf
|
||||
- VEXED
|
||||
# - VEXED
|
||||
- WAVED
|
||||
- YARD
|
||||
- HiTE
|
||||
- RPO
|
||||
- MEOWZiK
|
||||
- STONERD
|
||||
- TR
|
||||
- W4GN3R
|
||||
|
||||
@ -15,9 +15,10 @@ topic:
|
||||
11: [ 1869, 1815 ]
|
||||
12: [ 1788, 1719 ]
|
||||
13: [ 739, 1779 ]
|
||||
14: [ 951, 1796 ]
|
||||
14: [ 951, 1702 ]
|
||||
15: [ 1740, 1730 ]
|
||||
16: [ 1736, 1720 ]
|
||||
16: [ 1736, 1720 ]
|
||||
17: [ 1746, 1728 ]
|
||||
18: [ 1744, 1724 ]
|
||||
19: [ 1748, 1815 ]
|
||||
|
||||
29
config/scope/stb.yaml
Normal file
29
config/scope/stb.yaml
Normal file
@ -0,0 +1,29 @@
|
||||
role: console
|
||||
stb:
|
||||
endpoint: stb@amok.space
|
||||
#mailboxes: ["INBOX"]
|
||||
#mailboxes: ["INBOX", "Junk Mail", "Done"]
|
||||
mailboxes: ["Junk Mail"]
|
||||
#mailboxes: ["Done"]
|
||||
search-criteria: ["Your Share The Brutality digest", "noreply@sharethebrutality.info"]
|
||||
move-processed-to-mailbox:
|
||||
succeed: Processed/Succeed
|
||||
#succeed: Done
|
||||
failed: Processed/Failed
|
||||
suspicious: Processed/Suspicious
|
||||
sender:
|
||||
mailbox: noreply
|
||||
host: sharethebrutality.info
|
||||
subject: Your Share The Brutality digest
|
||||
db-type: nnmc
|
||||
regex:
|
||||
type-id: (?is)^\/forum\/topic\/(\d+)-
|
||||
who-genre: (?is)^(.*) created a topic in (Discographies|Death Metal|Old School Death|Other Genres|Your Own Rips)
|
||||
topics:
|
||||
discographies: 9
|
||||
death metal: 11
|
||||
old school death: 33
|
||||
other genres: 40
|
||||
your own rips: 14
|
||||
storage:
|
||||
filepath: c:\tmp\mdb\stb\
|
||||
55
go.mod
55
go.mod
@ -1,50 +1,47 @@
|
||||
module git.amok.space/yevhen/resource-scraper
|
||||
|
||||
go 1.23.1
|
||||
go 1.24
|
||||
|
||||
require (
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/emersion/go-imap/v2 v2.0.0-beta.5
|
||||
github.com/go-chi/chi/v5 v5.2.1
|
||||
github.com/go-chi/cors v1.2.1
|
||||
github.com/go-chi/jwtauth/v5 v5.3.1
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
|
||||
github.com/go-sql-driver/mysql v1.8.1
|
||||
github.com/golang-module/carbon/v2 v2.3.12
|
||||
github.com/go-sql-driver/mysql v1.9.0
|
||||
github.com/golang-module/carbon/v2 v2.4.1
|
||||
github.com/iancoleman/strcase v0.3.0
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0
|
||||
github.com/mewkiz/flac v1.0.12
|
||||
github.com/spf13/pflag v1.0.6
|
||||
github.com/spf13/viper v1.19.0
|
||||
golang.org/x/net v0.29.0
|
||||
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa
|
||||
golang.org/x/net v0.35.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
|
||||
github.com/fsnotify/fsnotify v1.8.0 // indirect
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
|
||||
github.com/lestrrat-go/httpcc v1.0.1 // indirect
|
||||
github.com/lestrrat-go/httprc v1.0.4 // indirect
|
||||
github.com/lestrrat-go/iter v1.0.2 // indirect
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.20 // indirect
|
||||
github.com/lestrrat-go/option v1.0.1 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/icza/bitio v1.1.0 // indirect
|
||||
github.com/magiconair/properties v1.8.9 // indirect
|
||||
github.com/mewkiz/pkg v0.0.0-20241223220703-7f3c7df797ff // indirect
|
||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
github.com/sagikazarmark/locafero v0.7.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/segmentio/asm v1.2.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/spf13/afero v1.12.0 // indirect
|
||||
github.com/spf13/cast v1.7.1 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.27.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||
golang.org/x/sys v0.25.0 // indirect
|
||||
golang.org/x/text v0.18.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/text v0.22.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
169
go.sum
169
go.sum
@ -1,152 +1,173 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
|
||||
github.com/emersion/go-imap/v2 v2.0.0-beta.5 h1:H3858DNmBuXyMK1++YrQIRdpKE1MwBc+ywBtg3n+0wA=
|
||||
github.com/emersion/go-imap/v2 v2.0.0-beta.5/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
|
||||
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
||||
github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
|
||||
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
|
||||
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||
github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
|
||||
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
|
||||
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
|
||||
github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
||||
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
|
||||
github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||
github.com/go-chi/jwtauth/v5 v5.3.1 h1:1ePWrjVctvp1tyBq5b/2ER8Th/+RbYc7x4qNsc5rh5A=
|
||||
github.com/go-chi/jwtauth/v5 v5.3.1/go.mod h1:6Fl2RRmWXs3tJYE1IQGX81FsPoGqDwq9c15j52R5q80=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/go-sql-driver/mysql v1.9.0 h1:Y0zIbQXhQKmQgTp44Y1dp3wTXcn804QoTptLZT1vtvo=
|
||||
github.com/go-sql-driver/mysql v1.9.0/go.mod h1:pDetrLJeA3oMujJuvXc8RJoasr589B6A9fwzD3QMrqw=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
|
||||
github.com/golang-module/carbon/v2 v2.3.12 h1:VC1DwN1kBwJkh5MjXmTFryjs5g4CWyoM8HAHffZPX/k=
|
||||
github.com/golang-module/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/golang-module/carbon/v2 v2.4.1 h1:cYUD8T+rHeX+qIybGYpnJ8I90F10dvyEF67VNOO+zZM=
|
||||
github.com/golang-module/carbon/v2 v2.4.1/go.mod h1:1jP9AZ4k2+lmfgY/wZgmtsN52VcHC5YuPM6varKDTkM=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
|
||||
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
|
||||
github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0=
|
||||
github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A=
|
||||
github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k=
|
||||
github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA=
|
||||
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
|
||||
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
|
||||
github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
|
||||
github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
|
||||
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
|
||||
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
|
||||
github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8=
|
||||
github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
|
||||
github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
|
||||
github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.20 h1:sAgXuWS/t8ykxS9Bi2Qtn5Qhpakw1wrcjxChudjolCc=
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.20/go.mod h1:UlCSmKqw+agm5BsOBfEAbTvKsEApaGNqHAEUTv5PJC4=
|
||||
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
|
||||
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
|
||||
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
|
||||
github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM=
|
||||
github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mewkiz/flac v1.0.12 h1:5Y1BRlUebfiVXPmz7hDD7h3ceV2XNrGNMejNVjDpgPY=
|
||||
github.com/mewkiz/flac v1.0.12/go.mod h1:1UeXlFRJp4ft2mfZnPLRpQTd7cSjb/s17o7JQzzyrCA=
|
||||
github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14/go.mod h1:QYCFBiH5q6XTHEbWhR0uhR3M9qNPoD2CSQzr0g75kE4=
|
||||
github.com/mewkiz/pkg v0.0.0-20241223220703-7f3c7df797ff h1:ts1OPi6zzzHE4nosvaXfkvhVNHeBghtxwwJL/PBsVHk=
|
||||
github.com/mewkiz/pkg v0.0.0-20241223220703-7f3c7df797ff/go.mod h1:dpOqQ/SkbEbPklAuIQqDGHc6K6kygt0L1XYa/IMlQeI=
|
||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
|
||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
|
||||
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
|
||||
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
|
||||
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
|
||||
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
|
||||
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
|
||||
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
|
||||
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
|
||||
github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
|
||||
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
|
||||
github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
|
||||
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
|
||||
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
|
||||
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
|
||||
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4=
|
||||
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
|
||||
golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
|
||||
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
|
||||
golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
|
||||
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
|
||||
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
|
||||
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@ -2,7 +2,6 @@ package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"log/slog"
|
||||
@ -15,7 +14,7 @@ import (
|
||||
|
||||
func setClient(url string) (*http.Response, error) {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
@ -27,9 +26,12 @@ func setClient(url string) (*http.Response, error) {
|
||||
|
||||
func HTMLSourceFromURL(url string) (*html.Node, error) {
|
||||
resp, err := setClient(url)
|
||||
defer func(Body io.ReadCloser) {
|
||||
fmt.Printf("%v\n", Body == nil)
|
||||
if resp == nil {
|
||||
slog.Error("client return nil response", "err", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func(Body io.ReadCloser) {
|
||||
err = Body.Close()
|
||||
if err != nil {
|
||||
slog.Error("closing response body", "err", err)
|
||||
|
||||
19
helper/sugar/console.go
Normal file
19
helper/sugar/console.go
Normal file
@ -0,0 +1,19 @@
|
||||
package sugar
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
)
|
||||
|
||||
func LogSuccess(msg string) {
|
||||
fmt.Printf("%s: %s\n", aurora.Green("SUCCESS"), aurora.Yellow(msg))
|
||||
}
|
||||
|
||||
func LogError(msg string) {
|
||||
fmt.Printf("%s: %s\n", aurora.Red("ERROR"), aurora.White(msg))
|
||||
}
|
||||
|
||||
func LogWarning(msg string) {
|
||||
fmt.Printf("%s: %s\n", aurora.Magenta("WARNING"), aurora.White(msg))
|
||||
}
|
||||
11
helper/sugar/env.go
Normal file
11
helper/sugar/env.go
Normal file
@ -0,0 +1,11 @@
|
||||
package sugar
|
||||
|
||||
import (
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func IsDev() bool {
|
||||
return viper.GetString(constant.FlagEnv) == constant.DefaultEnvDev
|
||||
}
|
||||
63
helper/sugar/fs.go
Normal file
63
helper/sugar/fs.go
Normal file
@ -0,0 +1,63 @@
|
||||
package sugar
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func EnsureDir(dirPath ...string) (string, error) {
|
||||
outPath := filepath.Join(dirPath...)
|
||||
if len(outPath) < 3 {
|
||||
return "", fmt.Errorf("the path is short to be a folder: %s", outPath)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(outPath); os.IsNotExist(err) {
|
||||
var dirMod uint64
|
||||
if dirMod, err = strconv.ParseUint("0775", 8, 32); err == nil {
|
||||
err = os.Mkdir(outPath, os.FileMode(dirMod))
|
||||
}
|
||||
|
||||
if err != nil && !os.IsExist(err) {
|
||||
slog.Error("error creating tmp dir", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
return outPath, nil
|
||||
}
|
||||
|
||||
func WriteDataToTmpFile(content, fp string) (int, string, error) {
|
||||
dir, err := EnsureDir(os.TempDir(), "mdb")
|
||||
if err != nil {
|
||||
slog.Error("EnsureDir", "err", err.Error())
|
||||
return 0, dir, err
|
||||
}
|
||||
|
||||
fp = filepath.Join(dir, fp)
|
||||
|
||||
f, err := os.Create(fp)
|
||||
if err != nil {
|
||||
slog.Error("CreateFile", "e", err)
|
||||
return 0, fp, err
|
||||
}
|
||||
|
||||
filesize, err := f.WriteString(content)
|
||||
if err != nil {
|
||||
slog.Error("WriteString", "e", err)
|
||||
return 0, fp, err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
slog.Error("CloseFile", "e", err)
|
||||
return 0, fp, err
|
||||
}
|
||||
|
||||
return filesize, fp, nil
|
||||
}
|
||||
|
||||
/*
|
||||
REMOVED_FROM_MMT_DUE_TO_REDOING_OF_THE_EVENT_LOG
|
||||
*/
|
||||
8
helper/sugar/string.go
Normal file
8
helper/sugar/string.go
Normal file
@ -0,0 +1,8 @@
|
||||
package sugar
|
||||
|
||||
import "regexp"
|
||||
|
||||
// SqueezeLine remove extra spaces and line breaks from a string
|
||||
func SqueezeLine(line string) string {
|
||||
return regexp.MustCompile(`(?m)[\n\r\t\s]+`).ReplaceAllString(line, " ")
|
||||
}
|
||||
28
helper/thither/thither.go
Normal file
28
helper/thither/thither.go
Normal file
@ -0,0 +1,28 @@
|
||||
package thither
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func FieldValueToIntSlice[T interface{}](object []T, key string) []int {
|
||||
fields := make([]int, len(object))
|
||||
|
||||
for i, el := range object {
|
||||
immutable := reflect.ValueOf(el)
|
||||
fields[i] = immutable.FieldByName(key).Interface().(int)
|
||||
}
|
||||
|
||||
return fields
|
||||
}
|
||||
|
||||
func FieldValueToStrSlice[T interface{}](object []T, key string) []string {
|
||||
fields := make([]string, len(object))
|
||||
|
||||
for i, el := range object {
|
||||
immutable := reflect.ValueOf(el)
|
||||
fields[i] = strconv.Itoa(immutable.FieldByName(key).Interface().(int))
|
||||
}
|
||||
|
||||
return fields
|
||||
}
|
||||
@ -3,7 +3,10 @@ package scraper
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/iancoleman/strcase"
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/internal/db"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/handler"
|
||||
@ -12,8 +15,22 @@ import (
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func isAllowScope() bool {
|
||||
scopesAllow := viper.GetStringSlice("scope.allow")
|
||||
scopeEnabled := viper.GetString(constant.FlagScopeEnable)
|
||||
|
||||
return slices.Contains(scopesAllow, scopeEnabled)
|
||||
}
|
||||
|
||||
func init() {
|
||||
strcase.ConfigureAcronym("stb", "STB")
|
||||
}
|
||||
|
||||
func Bootstrap() {
|
||||
//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
|
||||
if !isAllowScope() {
|
||||
fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml"))
|
||||
return
|
||||
}
|
||||
|
||||
dbase := db.New()
|
||||
repos := repository.New(dbase)
|
||||
@ -23,10 +40,8 @@ func Bootstrap() {
|
||||
switch viper.GetString("role") {
|
||||
case constant.RoleConsole:
|
||||
fmt.Printf("init console console: %s\n", handlers.InitConsole())
|
||||
break
|
||||
case constant.RoleWeb:
|
||||
fmt.Printf("who: %s\n", handlers.InitRoutes())
|
||||
///http.Run()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,7 +28,8 @@ Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
|
||||
|
||||
const (
|
||||
version = "0.1"
|
||||
defaultConfigPath = "config/default"
|
||||
defaultConfigPath = constant.DefaultConfigPath
|
||||
defaultConfigEnv = constant.DefaultEnvProd
|
||||
)
|
||||
|
||||
func ParseFlags() {
|
||||
@ -37,8 +38,11 @@ func ParseFlags() {
|
||||
flag.Bool(constant.FlagVersionShort, false, "")
|
||||
flag.Bool(constant.FlagVersion, false, "")
|
||||
flag.Bool(constant.FlagDebug, false, "")
|
||||
flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location used for the program")
|
||||
flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location")
|
||||
flag.String(constant.FlagScopeEnable, "", "")
|
||||
flag.String(constant.FlagSingleUri, "", "")
|
||||
flag.String(constant.FlagEnv, defaultConfigEnv, "")
|
||||
flag.String("create", defaultConfigEnv, "used to create e.g. scope etc.")
|
||||
|
||||
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
|
||||
pflag.Parse()
|
||||
|
||||
23
internal/logging/logging.go
Normal file
23
internal/logging/logging.go
Normal file
@ -0,0 +1,23 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
func custom() {
|
||||
//logger := log.New(os.Stderr, "INFO\t", log.LstdFlags|log.Lmicroseconds|log.Lshortfile)
|
||||
}
|
||||
|
||||
// New check this later https://github.com/hedzr/logg
|
||||
func New() {
|
||||
handlerOpts := &slog.HandlerOptions{
|
||||
Level: slog.LevelDebug,
|
||||
//AddSource: true,
|
||||
}
|
||||
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stderr, handlerOpts))
|
||||
//.WithAttrs([]slog.Attr{slog.String("app_version", "v0.0.1")})
|
||||
|
||||
slog.SetDefault(logger)
|
||||
}
|
||||
172
internal/mail/mail.go
Normal file
172
internal/mail/mail.go
Normal file
@ -0,0 +1,172 @@
|
||||
package mail
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/emersion/go-imap/v2"
|
||||
client "github.com/emersion/go-imap/v2/imapclient"
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/helper/sugar"
|
||||
)
|
||||
|
||||
type EmailService struct {
|
||||
User string
|
||||
pass string
|
||||
Err error
|
||||
client *client.Client
|
||||
Mailboxes []*imap.SelectData
|
||||
Messages []*client.FetchMessageBuffer
|
||||
}
|
||||
|
||||
func (s *EmailService) CheckErr(msg string, err error) bool {
|
||||
if err != nil {
|
||||
if sugar.IsDev() {
|
||||
sugar.LogError(err.Error())
|
||||
}
|
||||
s.Err = err
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *EmailService) success(msg string) {
|
||||
if sugar.IsDev() {
|
||||
sugar.LogSuccess(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EmailService) Warn(msg string) {
|
||||
if sugar.IsDev() {
|
||||
sugar.LogWarning(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EmailService) Connect() {
|
||||
box := strings.Split(s.User, "@")
|
||||
mail := viper.GetStringMapString("mail." + box[1])
|
||||
|
||||
conn, err := client.DialTLS(mail["dial-tls"], nil)
|
||||
if s.CheckErr("DialTLS", err) {
|
||||
return
|
||||
}
|
||||
|
||||
s.client = conn
|
||||
s.pass = mail[s.User]
|
||||
s.success("connected to " + mail["dial-tls"])
|
||||
|
||||
//defer s.Logout()
|
||||
}
|
||||
|
||||
func (s *EmailService) Login() {
|
||||
s.Connect()
|
||||
|
||||
err := s.client.Login(s.User, s.pass).Wait()
|
||||
if s.CheckErr("Login", err) {
|
||||
return
|
||||
}
|
||||
s.success(s.User + " logged")
|
||||
}
|
||||
|
||||
func (s *EmailService) ListMailboxes(mailboxes []string) {
|
||||
selectOptions := &imap.SelectOptions{ReadOnly: true}
|
||||
|
||||
for _, mailbox := range mailboxes {
|
||||
mbox, err := s.client.Select(mailbox, selectOptions).Wait()
|
||||
if s.CheckErr("Listing mailbox", err) {
|
||||
return
|
||||
}
|
||||
s.Mailboxes = append(s.Mailboxes, mbox)
|
||||
}
|
||||
|
||||
if len(s.Mailboxes) == 0 {
|
||||
s.Warn(sugar.SqueezeLine("mailboxes " + strings.Join(mailboxes, ", ") + " not found"))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EmailService) ListMessages(mailboxes []string, criteria *imap.SearchCriteria) {
|
||||
for _, mailbox := range mailboxes {
|
||||
mbox, err := s.client.Select(mailbox, nil).Wait()
|
||||
if s.CheckErr("Listing mailbox", err) {
|
||||
continue
|
||||
}
|
||||
|
||||
seqs, err := s.client.UIDSearch(criteria, nil).Wait()
|
||||
if s.CheckErr("UIDSearch", err) {
|
||||
return
|
||||
}
|
||||
|
||||
seqSet := imap.SeqSet{}
|
||||
seqSet.AddRange(1, mbox.NumMessages)
|
||||
if len(seqs.AllUIDs()) == 0 {
|
||||
s.Warn("no messages found in mailbox: " + mailbox)
|
||||
continue
|
||||
}
|
||||
s.success(fmt.Sprintf("Search complete, found %d messages: %+v", len(seqs.AllUIDs()), mbox.UIDNext))
|
||||
|
||||
fetchOptions := &imap.FetchOptions{
|
||||
Envelope: true,
|
||||
Flags: true,
|
||||
BodyStructure: &imap.FetchItemBodyStructure{Extended: true},
|
||||
BodySection: []*imap.FetchItemBodySection{{Peek: true, Part: []int{2}}},
|
||||
}
|
||||
|
||||
messages, err := s.client.Fetch(seqSet, fetchOptions).Collect()
|
||||
if s.CheckErr("Fetch", err) {
|
||||
continue
|
||||
}
|
||||
|
||||
s.Messages = messages
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EmailService) LogOut() {
|
||||
err := s.client.Logout().Wait()
|
||||
if !s.CheckErr("failed to logout", err) {
|
||||
s.success("logged out successfully")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EmailService) CreateMailbox(mailboxName string) {
|
||||
s.client.Create(mailboxName, nil)
|
||||
}
|
||||
|
||||
func (s *EmailService) MailboxesList() {
|
||||
listCmd := s.client.List("", "%", &imap.ListOptions{
|
||||
ReturnStatus: &imap.StatusOptions{
|
||||
NumMessages: true,
|
||||
NumUnseen: true,
|
||||
},
|
||||
})
|
||||
|
||||
for {
|
||||
mbox := listCmd.Next()
|
||||
if mbox == nil {
|
||||
break
|
||||
}
|
||||
|
||||
fmt.Printf("%+v\n", mbox)
|
||||
}
|
||||
|
||||
if err := listCmd.Close(); err != nil {
|
||||
log.Fatalf("LIST command failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EmailService) MoveMessageToMailbox(msg *client.FetchMessageBuffer, status string) bool {
|
||||
movable := imap.SeqSet{}
|
||||
movable.AddNum(msg.SeqNum)
|
||||
mailbox := viper.GetStringMapString("stb.move-processed-to-mailbox")
|
||||
wait, err := s.client.Move(movable, mailbox[status]).Wait()
|
||||
if s.CheckErr("Moving to archive", err) {
|
||||
return false
|
||||
}
|
||||
|
||||
fmt.Printf("Message %s moved, srcUIDs: %s, dstUIDs: %s\n", aurora.White(msg.Envelope.MessageID), aurora.Yellow(wait.SourceUIDs), aurora.Yellow(wait.DestUIDs))
|
||||
|
||||
return true
|
||||
}
|
||||
26
pkg/handler/console.go
Normal file
26
pkg/handler/console.go
Normal file
@ -0,0 +1,26 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func (h *Handler) Console() string {
|
||||
viper.SetDefault("env", "devel")
|
||||
cmd := viper.GetString("create")
|
||||
cmdList := viper.Sub("console.cmd")
|
||||
allowCreate := cmdList.GetStringSlice("create")
|
||||
|
||||
if !slices.Contains(allowCreate, cmd) {
|
||||
fmt.Printf("%s Not allowed command %s used\n", aurora.BgMagenta("[WARN]"), aurora.Magenta(cmd))
|
||||
return ""
|
||||
}
|
||||
|
||||
fmt.Printf("%v\n", slices.Contains(allowCreate, cmd))
|
||||
|
||||
return viper.GetString("env")
|
||||
|
||||
}
|
||||
@ -1,6 +1,9 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"github.com/iancoleman/strcase"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/service"
|
||||
@ -16,14 +19,13 @@ func New(services *service.Service) *Handler {
|
||||
}
|
||||
|
||||
func (h *Handler) InitConsole() string {
|
||||
switch viper.GetString(constant.CfgKeyScopeEnable) {
|
||||
case constant.ScopeRuTracker:
|
||||
return h.rutracker()
|
||||
case constant.ScopePrescene:
|
||||
return h.prescene()
|
||||
}
|
||||
methodName := strcase.ToCamel(viper.GetString(constant.FlagScopeEnable))
|
||||
|
||||
return "no scope chosen"
|
||||
immutable := reflect.ValueOf(h)
|
||||
method := immutable.MethodByName(methodName)
|
||||
v := method.Call(nil)
|
||||
|
||||
return methodName + " launched, " + v[0].String() + "\n"
|
||||
}
|
||||
|
||||
func (h *Handler) InitRoutes() string {
|
||||
|
||||
36
pkg/handler/info.go
Normal file
36
pkg/handler/info.go
Normal file
@ -0,0 +1,36 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/mewkiz/flac"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) Info() string {
|
||||
md5()
|
||||
|
||||
fmt.Printf("%s: %s; %s: %s\n",
|
||||
aurora.Cyan("ENV"),
|
||||
viper.GetString(constant.FlagEnv), aurora.Cyan("SCOPE"), viper.GetString(constant.FlagScopeEnable))
|
||||
return "info"
|
||||
}
|
||||
|
||||
func md5() {
|
||||
stream, err := flac.ParseFile("C:\\arm.amok.space\\.incoming\\Ancient Storm\\Forever and Never (2024)\\06 Old Mountain.flac")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer stream.Close()
|
||||
|
||||
fmt.Printf("unencoded audio md5sum: %032x\n", stream.Info.MD5sum[:])
|
||||
fmt.Printf("Total number of inter-channel samples in the stream: %+v\n", stream.Info.NSamples)
|
||||
for i, block := range stream.Blocks {
|
||||
fmt.Printf("block %d: %v\n", i, block.Type)
|
||||
}
|
||||
}
|
||||
5
pkg/handler/metal-archives.go
Normal file
5
pkg/handler/metal-archives.go
Normal file
@ -0,0 +1,5 @@
|
||||
package handler
|
||||
|
||||
func (h *Handler) MetalArchives() string {
|
||||
return "MetalArchives................."
|
||||
}
|
||||
@ -12,7 +12,7 @@ import (
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) prescene() string {
|
||||
func (h *Handler) Prescene() string {
|
||||
pagesToScrape := []string{"1"}
|
||||
levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
|
||||
if levels > 1 {
|
||||
|
||||
@ -8,7 +8,7 @@ import (
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func (h *Handler) rutracker() string {
|
||||
func (h *Handler) Rutracker() string {
|
||||
key := fmt.Sprintf("topic.%v", time.Now().Hour())
|
||||
topics := viper.GetStringSlice(key)
|
||||
|
||||
|
||||
36
pkg/handler/stb.go
Normal file
36
pkg/handler/stb.go
Normal file
@ -0,0 +1,36 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) STB() string {
|
||||
endpoint := fmt.Sprintf("%s.%s", constant.ScopeShareTheBrutality, constant.CfgKeyEndpoint)
|
||||
endpoint = viper.GetString(endpoint)
|
||||
|
||||
if endpoint == "" {
|
||||
slog.Error("getting endpoint from config", "err", errors.New("no endpoint provided"))
|
||||
return "stb"
|
||||
}
|
||||
|
||||
es, ms := h.services.ShareTheBrutality.GetMail(endpoint)
|
||||
|
||||
//fmt.Printf("%+v\n", es)
|
||||
ms.LogOut()
|
||||
|
||||
for _, record := range es {
|
||||
fmt.Printf("%s %d: %s [#%s]\n", record.Created, record.Id, record.Title, record.Releaser)
|
||||
}
|
||||
|
||||
/*if err != nil {
|
||||
slog.Error("error occurred while getting topic: ", "err", err)
|
||||
}*/
|
||||
|
||||
return fmt.Sprintf("Added %d records\n", len(es))
|
||||
}
|
||||
@ -1,11 +1,13 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-shiori/dom"
|
||||
@ -30,79 +32,103 @@ func NewPresceneRepository(db *sqlx.DB) *Prescene {
|
||||
func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
||||
entries := make([]model.ExternalSources, 0)
|
||||
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
||||
//scope := viper.GetString(constant.CfgKeyScopeEnable)
|
||||
tags := viper.GetStringMapStringSlice("groups.tags")
|
||||
|
||||
for _, t := range pageNumbers {
|
||||
if t != "1" {
|
||||
endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
|
||||
}
|
||||
doc, err := parser.HTMLSourceFromURL(endpoint)
|
||||
//doc, err := parser.HTMLSourceFromURL("https://mdb.amok.space/$/scnlog.html")
|
||||
|
||||
if err != nil {
|
||||
slog.Error("Parse error", "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if doc == nil {
|
||||
slog.Warn("Document is nil", "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
|
||||
var es model.ExternalSources
|
||||
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
|
||||
|
||||
title := dom.QuerySelector(item, ".title")
|
||||
if title != nil {
|
||||
anchor := dom.QuerySelector(title, "h1 > a")
|
||||
if anchor != nil {
|
||||
es.Type = constant.ScopePrescene
|
||||
es.Title = dom.GetAttribute(anchor, "title")
|
||||
es.ExSource = dom.GetAttribute(anchor, "href")
|
||||
pattern := regexp.MustCompile(`(?is)-(\w+)$`)
|
||||
es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
|
||||
|
||||
for flag, groups := range tags {
|
||||
if slices.Contains(groups, es.Releaser) {
|
||||
es.A = flag
|
||||
es.H = flag
|
||||
columns = append(columns, "a", "h")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if es.A == constant.TagIgnore {
|
||||
slog.Info("Skipped", "releaser", es.Releaser)
|
||||
continue
|
||||
}
|
||||
|
||||
localtime := dom.QuerySelector(title, "small > span.localtime")
|
||||
if localtime != nil {
|
||||
lc := dom.GetAttribute(localtime, "data-lttime")
|
||||
es.Created = carbon.Parse(lc)
|
||||
}
|
||||
uri := viper.GetString(constant.FlagSingleUri)
|
||||
if uri != "" {
|
||||
url := fmt.Sprintf("%s/%s", strings.Trim(endpoint, "/"), strings.Trim(uri, "/"))
|
||||
result, _ := parseUrl(url, s.db)
|
||||
entries = append(entries, result...)
|
||||
} else {
|
||||
for _, t := range pageNumbers {
|
||||
if t != "1" {
|
||||
endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
|
||||
}
|
||||
|
||||
cls := dom.GetAttribute(item, "class")
|
||||
pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
|
||||
es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
|
||||
//doc, err := parser.HTMLSourceFromURL("https://mdb.amok.space/$/scnlog.html")
|
||||
if result, err := parseUrl(endpoint, s.db); err == nil {
|
||||
entries = append(entries, result...)
|
||||
} else {
|
||||
slog.Error("parsing url", "err", err)
|
||||
}
|
||||
|
||||
//fmt.Println("====================== ", i, " ==============================")
|
||||
esModel := table.ExternalSources{Columns: columns}
|
||||
entry := esModel.InsertOnDuplicate(es, s.db)
|
||||
entries = append(entries, entry)
|
||||
|
||||
//fmt.Printf("%+v\n", entry)
|
||||
//fmt.Println("Sleeping...", j)
|
||||
time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
|
||||
}
|
||||
|
||||
//fmt.Println("Sleeping...", j)
|
||||
time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
|
||||
}
|
||||
|
||||
//fmt.Printf("scope: %v\n", scope)
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
|
||||
entries := make([]model.ExternalSources, 0)
|
||||
tags := viper.GetStringMapStringSlice("groups.tags")
|
||||
slog.Info("singleton", "url", endpoint)
|
||||
doc, err := parser.HTMLSourceFromURL(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if doc == nil {
|
||||
return nil, errors.New("document is nil")
|
||||
}
|
||||
var validID = regexp.MustCompile(`-\d+\/$`)
|
||||
|
||||
for i, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
|
||||
var es model.ExternalSources
|
||||
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
|
||||
|
||||
title := dom.QuerySelector(item, ".title")
|
||||
if title != nil {
|
||||
anchor := dom.QuerySelector(title, "h1 > a")
|
||||
if anchor != nil {
|
||||
es.Type = constant.ScopePrescene
|
||||
es.Title = dom.GetAttribute(anchor, "title")
|
||||
if es.Title == "Auto Draft" {
|
||||
slog.Info("Skipped", "title", es.Title)
|
||||
continue
|
||||
}
|
||||
|
||||
es.ExSource = dom.GetAttribute(anchor, "href")
|
||||
if validID.MatchString(es.ExSource) {
|
||||
continue
|
||||
}
|
||||
|
||||
pattern := regexp.MustCompile(`(?is)-(\w+)$`)
|
||||
es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
|
||||
|
||||
for flag, groups := range tags {
|
||||
if slices.Contains(groups, es.Releaser) {
|
||||
es.A = flag
|
||||
es.H = flag
|
||||
columns = append(columns, "a", "h")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if es.A == constant.TagIgnore {
|
||||
slog.Info("Skipped", "releaser", es.Releaser)
|
||||
continue
|
||||
}
|
||||
|
||||
localtime := dom.QuerySelector(title, "small > span.localtime")
|
||||
if localtime != nil {
|
||||
lc := dom.GetAttribute(localtime, "data-lttime")
|
||||
es.Created = carbon.Parse(lc)
|
||||
}
|
||||
}
|
||||
|
||||
cls := dom.GetAttribute(item, "class")
|
||||
pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
|
||||
es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
|
||||
|
||||
esModel := table.ExternalSources{Columns: columns}
|
||||
entry := esModel.InsertOnDuplicate(es, db)
|
||||
entries = append(entries, entry)
|
||||
|
||||
fmt.Println("====================== ", i, " ==============================")
|
||||
fmt.Printf("%+v\n", entry)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
@ -9,11 +9,13 @@ import (
|
||||
type Repository struct {
|
||||
_interface.Rutracker
|
||||
_interface.Prescene
|
||||
_interface.ShareTheBrutality
|
||||
}
|
||||
|
||||
func New(db *sqlx.DB) *Repository {
|
||||
return &Repository{
|
||||
Rutracker: NewRutrackerRepository(db),
|
||||
Prescene: NewPresceneRepository(db),
|
||||
Rutracker: NewRutrackerRepository(db),
|
||||
Prescene: NewPresceneRepository(db),
|
||||
ShareTheBrutality: NewShareTheBrutalityRepository(db),
|
||||
}
|
||||
}
|
||||
|
||||
154
pkg/repository/stb.go
Normal file
154
pkg/repository/stb.go
Normal file
@ -0,0 +1,154 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/emersion/go-imap/v2"
|
||||
"github.com/go-shiori/dom"
|
||||
"github.com/golang-module/carbon/v2"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/net/html"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/helper/parser"
|
||||
"git.amok.space/yevhen/resource-scraper/helper/sugar"
|
||||
"git.amok.space/yevhen/resource-scraper/internal/mail"
|
||||
_table "git.amok.space/yevhen/resource-scraper/pkg/repository/table"
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
type ShareTheBrutality struct {
|
||||
scope string
|
||||
EmailService mail.EmailService
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
func NewShareTheBrutalityRepository(db *sqlx.DB) *ShareTheBrutality {
|
||||
return &ShareTheBrutality{db: db, scope: constant.ScopeShareTheBrutality}
|
||||
}
|
||||
|
||||
func (s *ShareTheBrutality) GetMail(email string) ([]model.ExternalSources, *mail.EmailService) {
|
||||
s.EmailService = mail.EmailService{
|
||||
User: email,
|
||||
}
|
||||
|
||||
s.EmailService.Login()
|
||||
|
||||
mailboxes := fmt.Sprintf("%s.mailboxes", s.scope)
|
||||
criteria := fmt.Sprintf("%s.search-criteria", s.scope)
|
||||
searchCriteria := &imap.SearchCriteria{
|
||||
Text: viper.GetStringSlice(criteria),
|
||||
}
|
||||
//now := carbon.Now()
|
||||
s.EmailService.ListMessages(viper.GetStringSlice(mailboxes), searchCriteria)
|
||||
//box.CreateMailbox("INBOX/Processed")
|
||||
//s.EmailService.CreateMailbox("Processed/Succeed")
|
||||
//s.EmailService.CreateMailbox("Processed/Failed")
|
||||
//s.EmailService.CreateMailbox("Processed/Suspicious")
|
||||
//s.EmailService.MailboxesList()
|
||||
|
||||
entries := s.Processing(viper.GetStringMapString(fmt.Sprintf("%s.sender", s.scope)))
|
||||
|
||||
return entries, &s.EmailService
|
||||
}
|
||||
|
||||
func (s *ShareTheBrutality) Processing(sender map[string]string) []model.ExternalSources {
|
||||
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created", "fingerprint"}
|
||||
entriesBatched := make([]model.ExternalSources, 0)
|
||||
if len(s.EmailService.Messages) == 0 {
|
||||
return entriesBatched
|
||||
}
|
||||
|
||||
//tmpPath := viper.GetString(fmt.Sprintf("%s.storage.filepath", s.scope))
|
||||
dbType := viper.GetString(fmt.Sprintf("%s.db-type", s.scope))
|
||||
regexPatterns := viper.GetStringMapString(fmt.Sprintf("%s.regex", s.scope))
|
||||
topics := viper.GetStringMap(fmt.Sprintf("%s.topics", s.scope))
|
||||
|
||||
for _, msg := range s.EmailService.Messages {
|
||||
entries := make([]model.ExternalSources, 0)
|
||||
from := msg.Envelope.From[0]
|
||||
subject := msg.Envelope.Subject
|
||||
|
||||
if !(from.Mailbox == sender["mailbox"] && from.Host == sender["host"] && subject == sender["subject"]) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, section := range msg.BodySection {
|
||||
//sugar.WriteDataToTmpFile(msg.BodySection, tmpPath)
|
||||
|
||||
if section.Bytes != nil {
|
||||
doc, err := parser.HTMLSource(string(section.Bytes))
|
||||
if s.EmailService.CheckErr("parsing message body", err) {
|
||||
continue
|
||||
}
|
||||
|
||||
table := dom.QuerySelector(doc, "body > table:nth-of-type(1n) table:nth-of-type(1n) table:nth-of-type(2n) > tbody")
|
||||
if table == nil {
|
||||
s.EmailService.Warn("dom.QuerySelector had not queried any data, returned nil")
|
||||
continue
|
||||
}
|
||||
|
||||
var es model.ExternalSources
|
||||
for _, td := range dom.QuerySelectorAll(table, "tr > td:nth-child(2)") {
|
||||
anchor := dom.QuerySelector(td, "h2 > a")
|
||||
if anchor == nil {
|
||||
s.EmailService.Warn("dom.QuerySelector couldn't find title")
|
||||
continue
|
||||
}
|
||||
es.Title = sugar.SqueezeLine(dom.InnerHTML(anchor))
|
||||
|
||||
u, err := url.Parse(dom.GetAttribute(anchor, "href"))
|
||||
if s.EmailService.CheckErr("parsing url", err) {
|
||||
continue
|
||||
}
|
||||
|
||||
es.Fingerprint = u.RequestURI()
|
||||
pattern := regexp.MustCompile(regexPatterns["type-id"])
|
||||
typeIdMatch := pattern.FindStringSubmatch(es.Fingerprint)
|
||||
if len(typeIdMatch) != 2 {
|
||||
s.EmailService.Warn("Regexp => typeIdMatch not matched")
|
||||
continue
|
||||
}
|
||||
es.TypeId, _ = strconv.Atoi(typeIdMatch[1])
|
||||
|
||||
sourceData := dom.QuerySelector(td, "p:first-child")
|
||||
if sourceData == nil {
|
||||
s.EmailService.Warn("dom.QuerySelector couldn't find sourceData in paragraph")
|
||||
continue
|
||||
}
|
||||
sourceDataString := html.UnescapeString(sugar.SqueezeLine(dom.InnerHTML(sourceData)))
|
||||
pattern = regexp.MustCompile(regexPatterns["who-genre"])
|
||||
sourceDataMatch := pattern.FindStringSubmatch(sourceDataString)
|
||||
|
||||
if len(sourceDataMatch) != 3 {
|
||||
s.EmailService.Warn("Regexp => sourceData not matched")
|
||||
continue
|
||||
}
|
||||
|
||||
es.Releaser = sourceDataMatch[1]
|
||||
es.Created = carbon.Parse(msg.Envelope.Date.String())
|
||||
es.Type = dbType
|
||||
|
||||
genre := strings.ToLower(sourceDataMatch[2])
|
||||
es.TypeSubsectionId = topics[genre].(int)
|
||||
|
||||
entries = append(entries, es)
|
||||
}
|
||||
|
||||
result, status := _table.BatchInsertOnDuplicate(entries, s.db, columns)
|
||||
if status != constant.StatusFailed {
|
||||
entriesBatched = append(entriesBatched, result...)
|
||||
}
|
||||
|
||||
s.EmailService.MoveMessageToMailbox(msg, status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return entriesBatched
|
||||
}
|
||||
@ -2,10 +2,14 @@ package table
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/helper/thither"
|
||||
"git.amok.space/yevhen/resource-scraper/types/constant"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
@ -20,8 +24,6 @@ func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.D
|
||||
placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
|
||||
query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
|
||||
|
||||
//fmt.Printf("%s\n", query)
|
||||
|
||||
if rows, err := db.NamedQuery(query, &es); err == nil {
|
||||
for rows.Next() {
|
||||
es.Error = rows.StructScan(&es)
|
||||
@ -32,3 +34,45 @@ func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.D
|
||||
|
||||
return es
|
||||
}
|
||||
|
||||
func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, columns []string) ([]model.ExternalSources, string) {
|
||||
es := &ExternalSources{Columns: columns}
|
||||
typeIds := es.GetTypeIds(entries, db)
|
||||
var status string
|
||||
errCount := 0
|
||||
|
||||
for i := 0; i < len(entries); i++ {
|
||||
entry := es.InsertOnDuplicate(entries[i], db)
|
||||
if entry.Error != nil {
|
||||
slog.Error("insert/update entry", "err", entry.Error)
|
||||
errCount++
|
||||
}
|
||||
|
||||
if !slices.Contains(typeIds, entry.TypeId) {
|
||||
fmt.Printf("%s: %s\n", aurora.Green("ADDED"), aurora.White(entry.Title))
|
||||
}
|
||||
|
||||
entries[i] = es.InsertOnDuplicate(entries[i], db)
|
||||
}
|
||||
|
||||
if errCount == 0 {
|
||||
status = constant.StatusSucceed
|
||||
} else if errCount > 0 && errCount == len(entries) {
|
||||
status = constant.StatusFailed
|
||||
} else {
|
||||
status = constant.StatusSuspicious
|
||||
}
|
||||
|
||||
return entries, status
|
||||
}
|
||||
|
||||
func (f *ExternalSources) GetTypeIds(entries []model.ExternalSources, db *sqlx.DB) []int {
|
||||
var typeIds []int
|
||||
ids := thither.FieldValueToStrSlice(entries, "TypeId")
|
||||
query := fmt.Sprintf("SELECT type_id FROM %s WHERE `type` = '%s' AND type_id IN (%s) LIMIT %d", constant.ExternalSourcesTable, entries[0].Type, strings.Join(ids, ","), len(ids))
|
||||
err := db.Select(&typeIds, query)
|
||||
if err != nil {
|
||||
slog.Error("getting type ids", "err", err)
|
||||
}
|
||||
return typeIds
|
||||
}
|
||||
|
||||
7
pkg/service/info.go
Normal file
7
pkg/service/info.go
Normal file
@ -0,0 +1,7 @@
|
||||
package service
|
||||
|
||||
type InfoService struct{}
|
||||
|
||||
func NewInfoService() *InfoService {
|
||||
return &InfoService{}
|
||||
}
|
||||
1
pkg/service/metal-archives.go
Normal file
1
pkg/service/metal-archives.go
Normal file
@ -0,0 +1 @@
|
||||
package service
|
||||
@ -8,11 +8,15 @@ import (
|
||||
type Service struct {
|
||||
_interface.Rutracker
|
||||
_interface.Prescene
|
||||
_interface.Info
|
||||
_interface.ShareTheBrutality
|
||||
}
|
||||
|
||||
func New(repos *repository.Repository) *Service {
|
||||
return &Service{
|
||||
Rutracker: NewRutrackerService(repos.Rutracker),
|
||||
Prescene: NewPresceneService(repos.Prescene),
|
||||
Rutracker: NewRutrackerService(repos.Rutracker),
|
||||
Prescene: NewPresceneService(repos.Prescene),
|
||||
Info: NewInfoService(),
|
||||
ShareTheBrutality: NewShareTheBrutalityService(repos.ShareTheBrutality),
|
||||
}
|
||||
}
|
||||
|
||||
19
pkg/service/stb.go
Normal file
19
pkg/service/stb.go
Normal file
@ -0,0 +1,19 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"git.amok.space/yevhen/resource-scraper/internal/mail"
|
||||
"git.amok.space/yevhen/resource-scraper/types/interface"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
type ShareTheBrutalityService struct {
|
||||
repo _interface.ShareTheBrutality
|
||||
}
|
||||
|
||||
func NewShareTheBrutalityService(repo _interface.ShareTheBrutality) *ShareTheBrutalityService {
|
||||
return &ShareTheBrutalityService{repo: repo}
|
||||
}
|
||||
|
||||
func (stb *ShareTheBrutalityService) GetMail(email string) ([]model.ExternalSources, *mail.EmailService) {
|
||||
return stb.repo.GetMail(email)
|
||||
}
|
||||
@ -1,10 +1,11 @@
|
||||
package constant
|
||||
|
||||
const (
|
||||
ScopeRuTracker string = "rutracker"
|
||||
ScopePrescene string = "prescene"
|
||||
ScopeWeb string = "web"
|
||||
ScopeInfo string = "info"
|
||||
ScopeRuTracker string = "rutracker"
|
||||
ScopePrescene string = "prescene"
|
||||
ScopeWeb string = "web"
|
||||
ScopeInfo string = "info"
|
||||
ScopeShareTheBrutality string = "stb"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -16,3 +17,12 @@ const (
|
||||
TagIgnore string = "ignore"
|
||||
ExternalSourcesTable string = "external_sources"
|
||||
)
|
||||
|
||||
const (
|
||||
StatusSuccess string = "success"
|
||||
StatusError string = "error"
|
||||
|
||||
StatusSucceed string = "succeed"
|
||||
StatusSuspicious string = "suspicious"
|
||||
StatusFailed string = "failed"
|
||||
)
|
||||
|
||||
@ -8,4 +8,10 @@ const (
|
||||
FlagVersionShort = "v"
|
||||
FlagConfigFile = "config-file"
|
||||
FlagScopeEnable = "scope-enable"
|
||||
FlagSingleUri = "single-uri"
|
||||
FlagEnv = "env"
|
||||
|
||||
DefaultEnvProd = "prod"
|
||||
DefaultEnvDev = "devel"
|
||||
DefaultConfigPath = "config/default"
|
||||
)
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
package _interface
|
||||
|
||||
import "git.amok.space/yevhen/resource-scraper/types/model"
|
||||
import (
|
||||
"git.amok.space/yevhen/resource-scraper/internal/mail"
|
||||
"git.amok.space/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
type Rutracker interface {
|
||||
GetTopic(topics []string) ([]model.ExternalSources, error)
|
||||
@ -9,3 +12,11 @@ type Rutracker interface {
|
||||
type Prescene interface {
|
||||
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
|
||||
}
|
||||
|
||||
type ShareTheBrutality interface {
|
||||
GetMail(email string) ([]model.ExternalSources, *mail.EmailService)
|
||||
}
|
||||
|
||||
type MetalArchiveInterface interface{}
|
||||
|
||||
type Info interface{}
|
||||
|
||||
@ -21,7 +21,7 @@ type Type string
|
||||
)*/
|
||||
|
||||
type ExternalSources struct {
|
||||
Id int `json:"id" db:"id"`
|
||||
Id int64 `json:"id" db:"id"`
|
||||
Type string `json:"type" db:"type"`
|
||||
TypeId int `json:"type_id" db:"type_id"`
|
||||
Title string `json:"title" db:"title"`
|
||||
|
||||
45
types/resource/metal-archive.go
Normal file
45
types/resource/metal-archive.go
Normal file
@ -0,0 +1,45 @@
|
||||
package resource
|
||||
|
||||
import "github.com/golang-module/carbon/v2"
|
||||
|
||||
/*https://www.metal-archives.com/search/ajax-advanced/searching/albums/?bandName=&releaseTitle=&releaseYearFrom=2024&releaseMonthFrom=09&releaseYearTo=&releaseMonthTo=&country=&location=&releaseLabelName=&releaseCatalogNumber=&releaseIdentifiers=&releaseRecordingInfo=&releaseDescription=&releaseNotes=&genre=&sEcho=6&iColumns=4&sColumns=&iDisplayStart=1000&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&_=1726217541027*/
|
||||
|
||||
type AutoGenerated struct {
|
||||
Error string `json:"error"`
|
||||
ITotalRecords int `json:"iTotalRecords"`
|
||||
ITotalDisplayRecords int `json:"iTotalDisplayRecords"`
|
||||
SEcho int `json:"sEcho"`
|
||||
AaData [][]string `json:"aaData"`
|
||||
}
|
||||
|
||||
/*ALTER TABLE `go_tut_tokill`.`external_sources`
|
||||
CHANGE COLUMN `type` `type` ENUM('mmt', 'prescene', 'rutracker', 'locate', 'darkabyss', 'bfm', 'trash', 'nnmc', 'stb', 'bandcamp', 'deathgrind', 'ma') NULL DEFAULT NULL ;
|
||||
|
||||
*/
|
||||
|
||||
// AlbumQuery easy to debug https://www.freeformatter.com/url-parser-query-string-splitter.html
|
||||
type AlbumQuery struct {
|
||||
BandName string `param:"bandName"`
|
||||
ReleaseTitle string `param:"releaseTitle"`
|
||||
ReleaseYearFrom int `param:"releaseYearFrom"`
|
||||
ReleaseMonthFrom int `param:"releaseMonthFrom"`
|
||||
ReleaseYearTo int `param:"releaseYearTo"`
|
||||
ReleaseMonthTo int `param:"releaseMonthTo"`
|
||||
Country []string `param:"country"`
|
||||
Location string `param:"location"`
|
||||
ReleaseLabelName string `param:"releaseLabelName"`
|
||||
ReleaseCatalogNumber int `param:"releaseCatalogNumber"`
|
||||
ReleaseIdentifiers string `param:"releaseIdentifiers"`
|
||||
ReleaseRecordingInfo string `param:"releaseRecordingInfo"`
|
||||
ReleaseDescription string `param:"releaseDescription"`
|
||||
ReleaseNotes string `param:"releaseNotes"`
|
||||
Genre string `param:"genre"`
|
||||
ReleaseType []string `param:"releaseType"`
|
||||
SEcho string `param:"sEcho"`
|
||||
IColumns uint8 `param:"iColumns" default:"4"`
|
||||
SColumns string `param:"sColumns"`
|
||||
IDisplayStart uint8 `param:"iDisplayStart" default:"0"`
|
||||
IDisplayLength uint8 `param:"iDisplayLength" default:"200"`
|
||||
MDataProp uint8 `param:"mDataProp_%d"`
|
||||
Timestamp carbon.Carbon `param:"_"`
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user