add scraping the torrent topics rss files
This commit is contained in:
parent
20c4f25c55
commit
e587e645fc
@ -4,7 +4,7 @@ tmp_dir = "tmp"
|
||||
|
||||
[build]
|
||||
args_bin = []
|
||||
bin = "tmp\\main.exe"
|
||||
bin = "tmp\\main.exe --config-file ./config/default --scope rutracker"
|
||||
cmd = "go build -o ./tmp/main.exe ./cmd"
|
||||
delay = 1000
|
||||
exclude_dir = ["assets", "tmp", "vendor", "testdata"]
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -29,4 +29,5 @@ fabric.properties
|
||||
.idea/httpRequests
|
||||
.idea/caches/build_file_checksums.ser
|
||||
.env
|
||||
tmp/
|
||||
tmp/
|
||||
vendor/
|
||||
18
.idea/dataSources.xml
Normal file
18
.idea/dataSources.xml
Normal file
@ -0,0 +1,18 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
|
||||
<data-source source="LOCAL" name="@arm" uuid="e3965414-f0c4-4a05-af48-11e028c626da">
|
||||
<driver-ref>mariadb</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.mariadb.jdbc.Driver</jdbc-driver>
|
||||
<jdbc-url>jdbc:mariadb://10.14.88.1:6033/go_tut_tokill</jdbc-url>
|
||||
<jdbc-additional-properties>
|
||||
<property name="com.intellij.clouds.kubernetes.db.host.port" />
|
||||
<property name="com.intellij.clouds.kubernetes.db.enabled" value="false" />
|
||||
<property name="com.intellij.clouds.kubernetes.db.resource.type" value="Deployment" />
|
||||
<property name="com.intellij.clouds.kubernetes.db.container.port" />
|
||||
</jdbc-additional-properties>
|
||||
<working-dir>$ProjectFileDir$</working-dir>
|
||||
</data-source>
|
||||
</component>
|
||||
</project>
|
||||
10
.idea/inspectionProfiles/Project_Default.xml
Normal file
10
.idea/inspectionProfiles/Project_Default.xml
Normal file
@ -0,0 +1,10 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<profile version="1.0">
|
||||
<option name="myName" value="Project Default" />
|
||||
<inspection_tool class="GoDfaErrorMayBeNotNil" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<functions>
|
||||
<function importPath="git.amok.space/yevhen/resource-scraper/pkg/repository" name="fetch" />
|
||||
</functions>
|
||||
</inspection_tool>
|
||||
</profile>
|
||||
</component>
|
||||
22
cmd/main.go
22
cmd/main.go
@ -1,28 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
scraper "git.amok.space/yevhen/resource-scraper/internal"
|
||||
)
|
||||
|
||||
// init is invoked before main()
|
||||
func init() {
|
||||
// loads values from .env into the system
|
||||
if err := godotenv.Load(); err != nil {
|
||||
log.Print("No .env file found")
|
||||
}
|
||||
scraper.ParseFlags()
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Get the GREETING environment variable
|
||||
greeting, exists := os.LookupEnv("GREETING")
|
||||
|
||||
if exists {
|
||||
fmt.Println(greeting)
|
||||
}
|
||||
|
||||
fmt.Println("fin")
|
||||
scraper.Bootstrap()
|
||||
//fmt.Printf("%v\n", viper.Get("23").([]interface{}))
|
||||
}
|
||||
|
||||
5
config/default.yaml
Normal file
5
config/default.yaml
Normal file
@ -0,0 +1,5 @@
|
||||
db:
|
||||
driver: mysql
|
||||
# data_source_name: who:is@(10.14.88.1:6033)/go_tut_tokill
|
||||
data_source_name: theamok_mdb:2ilt3l72(!)r71a0a0d@(10.14.88.1:6033)/theamok_mdb
|
||||
|
||||
27
config/rutracker.yaml
Normal file
27
config/rutracker.yaml
Normal file
@ -0,0 +1,27 @@
|
||||
role: console
|
||||
endpoint: https://feed.rutracker.cc/atom/f/%s.atom
|
||||
topic:
|
||||
0: [ 1719, 1779 ]
|
||||
1: [ 1796, 1730 ]
|
||||
2: [ 1720, 1728 ]
|
||||
3: [ 1724, 1815 ]
|
||||
4: [ 1726, 1719 ]
|
||||
5: [ 2230, 1779 ]
|
||||
6: [ 1868, 1796 ]
|
||||
7: [ 1877, 1730 ]
|
||||
8: [ 1866, 1720 ]
|
||||
9: [ 1864, 1728 ]
|
||||
10: [ 1871, 1724 ]
|
||||
11: [ 1869, 1815 ]
|
||||
12: [ 1788, 1719 ]
|
||||
13: [ 739, 1779 ]
|
||||
14: [ 951, 1796 ]
|
||||
15: [ 1740, 1730 ]
|
||||
16: [ 1736, 1720 ]
|
||||
17: [ 1746, 1728 ]
|
||||
18: [ 1744, 1724 ]
|
||||
19: [ 1748, 1815 ]
|
||||
20: [ 1706, 1719 ]
|
||||
21: [ 1757, 1779 ]
|
||||
22: [ 1133, 739 ]
|
||||
23: [ 1755, 1756 ]
|
||||
31
go.mod
31
go.mod
@ -2,4 +2,33 @@ module git.amok.space/yevhen/resource-scraper
|
||||
|
||||
go 1.23.1
|
||||
|
||||
require github.com/joho/godotenv v1.5.1 // indirect
|
||||
require (
|
||||
github.com/go-chi/chi/v5 v5.1.0
|
||||
github.com/go-chi/cors v1.2.1
|
||||
github.com/go-sql-driver/mysql v1.8.1
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.19.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
87
go.sum
87
go.sum
@ -1,2 +1,85 @@
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||
github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
|
||||
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
|
||||
github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
|
||||
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
|
||||
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
|
||||
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
|
||||
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
|
||||
github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
|
||||
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
|
||||
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
7
helper/web/handlerReadiness.go
Normal file
7
helper/web/handlerReadiness.go
Normal file
@ -0,0 +1,7 @@
|
||||
package web
|
||||
|
||||
import "net/http"
|
||||
|
||||
func FallbackHandler(w http.ResponseWriter, r *http.Request) {
|
||||
respondWithJSON(w, http.StatusOK, struct{}{})
|
||||
}
|
||||
35
helper/web/json.go
Normal file
35
helper/web/json.go
Normal file
@ -0,0 +1,35 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type errorResponse struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) {
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
log.Printf("failed to marshal JSON response: %v\n", err)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
_, err = w.Write(data)
|
||||
if err != nil {
|
||||
log.Printf("failed to write data to output buffer: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func respondWithError(w http.ResponseWriter, code int, message string) {
|
||||
if code >= http.StatusInternalServerError {
|
||||
log.Println("responding with 5xx error:", message)
|
||||
}
|
||||
|
||||
respondWithJSON(w, code, errorResponse{Error: message})
|
||||
}
|
||||
31
internal/bootstrap.go
Normal file
31
internal/bootstrap.go
Normal file
@ -0,0 +1,31 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/internal/config"
|
||||
"git.amok.space/yevhen/resource-scraper/internal/db"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/handler"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/repository"
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/service"
|
||||
"git.amok.space/yevhen/resource-scraper/types"
|
||||
)
|
||||
|
||||
func Bootstrap() {
|
||||
config.New()
|
||||
|
||||
dbase := db.New()
|
||||
repos := repository.New(dbase)
|
||||
services := service.New(repos)
|
||||
handlers := handler.New(services)
|
||||
|
||||
switch viper.GetString("role") {
|
||||
case types.RoleConsole:
|
||||
fmt.Printf("init console console: %s\n", handlers.InitConsole())
|
||||
break
|
||||
}
|
||||
|
||||
///http.Run()
|
||||
}
|
||||
34
internal/config/config.go
Normal file
34
internal/config/config.go
Normal file
@ -0,0 +1,34 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func New() {
|
||||
configFilePath := viper.GetString("config-file")
|
||||
configDir := "./" + filepath.Dir(configFilePath)
|
||||
|
||||
viper.SetConfigName(filepath.Base(configFilePath))
|
||||
viper.AddConfigPath(configDir)
|
||||
|
||||
err := viper.ReadInConfig() // Find and read the config file
|
||||
if err != nil { // Handle errors reading the config file
|
||||
panic(fmt.Errorf("fatal error config file: %w", err))
|
||||
}
|
||||
|
||||
viper.SetDefault("ConfigDir", configDir)
|
||||
scope := viper.GetString("scope")
|
||||
if scope != "" {
|
||||
viper.SetConfigName(scope)
|
||||
viper.AddConfigPath(configDir)
|
||||
err := viper.MergeInConfig()
|
||||
if err != nil {
|
||||
log.Fatalf("fatal error config file: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
24
internal/db/connector.go
Normal file
24
internal/db/connector.go
Normal file
@ -0,0 +1,24 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func New() *sqlx.DB {
|
||||
drv := viper.GetString("db.driver")
|
||||
dsn := viper.GetString("db.data_source_name")
|
||||
|
||||
db, err := sqlx.Connect(drv, dsn)
|
||||
if err != nil {
|
||||
panic("Failed to connect to the database: " + err.Error())
|
||||
}
|
||||
|
||||
// Verify the connection to the database is still alive
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
panic("Failed to ping the database: " + err.Error())
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
55
internal/flag.go
Normal file
55
internal/flag.go
Normal file
@ -0,0 +1,55 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"runtime"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
const usage = `Music Database (MDB) server/cli craftware'
|
||||
|
||||
Usage:
|
||||
mdb [command] -c [config-file-path]
|
||||
|
||||
Commands:
|
||||
-help, -h Print this message
|
||||
-version -v Print version.
|
||||
-config -c Set configuration file without extension. It is %s by default.
|
||||
-section -s Select section e.g.: rutracker, bandcamp etc.
|
||||
|
||||
The Go runtime version: %s
|
||||
Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
|
||||
|
||||
const (
|
||||
maxPasswordLength = 20
|
||||
version = "0.1"
|
||||
defaultConfigPath = "./config/default"
|
||||
)
|
||||
|
||||
func ParseFlags() {
|
||||
flag.Bool("h", false, "")
|
||||
flag.Bool("help", false, "")
|
||||
flag.Bool("v", false, "")
|
||||
flag.Bool("version", false, "")
|
||||
flag.Bool("debug", false, "")
|
||||
flag.String("config-file", defaultConfigPath, "config file location used for the program")
|
||||
flag.String("scope", "", "")
|
||||
|
||||
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
|
||||
pflag.Parse()
|
||||
err := viper.BindPFlags(pflag.CommandLine)
|
||||
if err != nil {
|
||||
log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
|
||||
}
|
||||
|
||||
if viper.GetBool("h") || viper.GetBool("help") {
|
||||
msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
|
||||
fmt.Println(msg)
|
||||
} else if viper.GetBool("v") || viper.GetBool("version") {
|
||||
fmt.Println("MDB version", version)
|
||||
}
|
||||
}
|
||||
54
internal/http/http.go
Normal file
54
internal/http/http.go
Normal file
@ -0,0 +1,54 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/cors"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/helper/web"
|
||||
)
|
||||
|
||||
/*type Server struct {
|
||||
router *chi.Mux
|
||||
}*/
|
||||
|
||||
/*type Routes struct {
|
||||
Api *chi.Mux
|
||||
}*/
|
||||
|
||||
/*type Handler func(w http.ResponseWriter, r *http.Request) error
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if err := h(w, r); err != nil {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
|
||||
_, err := w.Write([]byte("bad"))
|
||||
if err != nil {
|
||||
log.Fatalf("Error starting HTTP server: %v", err)
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
func Run() {
|
||||
port := "19576"
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(cors.Handler(cors.Options{
|
||||
AllowedOrigins: []string{"https://*", "http://*"},
|
||||
AllowedMethods: []string{"GET", "POST"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposedHeaders: []string{"Link"},
|
||||
AllowCredentials: false,
|
||||
MaxAge: 300,
|
||||
}))
|
||||
|
||||
r.Get("/", web.FallbackHandler)
|
||||
|
||||
log.Printf("Server starting on port %v", port)
|
||||
err := http.ListenAndServe("localhost:"+port, r)
|
||||
if err != nil {
|
||||
log.Fatalf("Error starting HTTP server: %v", err)
|
||||
}
|
||||
}
|
||||
28
pkg/handler/handler.go
Normal file
28
pkg/handler/handler.go
Normal file
@ -0,0 +1,28 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/service"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
services *service.Service
|
||||
}
|
||||
|
||||
func New(services *service.Service) *Handler {
|
||||
return &Handler{services: services}
|
||||
}
|
||||
|
||||
func (h *Handler) InitConsole() string {
|
||||
return h.rutracker()
|
||||
}
|
||||
|
||||
//func (h *Handler) Base(services *service.Service) *Handler {
|
||||
// return &Handler{services: services}
|
||||
//}
|
||||
|
||||
//func (h *Handler) InitApi() *chi.Mux {
|
||||
// api := chi.NewRouter()
|
||||
// api.Get("/", web.ApiFallbackHandler)
|
||||
//
|
||||
// return api
|
||||
//}
|
||||
21
pkg/handler/rutracker.go
Normal file
21
pkg/handler/rutracker.go
Normal file
@ -0,0 +1,21 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func (h *Handler) rutracker() string {
|
||||
key := fmt.Sprintf("topic.%v", time.Now().Hour())
|
||||
topics := viper.GetStringSlice(key)
|
||||
|
||||
err := h.services.Rutracker.GetTopic(topics)
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting topic: ", err.Error())
|
||||
}
|
||||
|
||||
return "rt"
|
||||
}
|
||||
23
pkg/repository/repository.go
Normal file
23
pkg/repository/repository.go
Normal file
@ -0,0 +1,23 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/types"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
types.Rutracker
|
||||
}
|
||||
|
||||
func New(db *sqlx.DB) *Repository {
|
||||
switch viper.GetString("scope") {
|
||||
case types.RuTracker:
|
||||
return &Repository{
|
||||
Rutracker: NewRutracker(db),
|
||||
}
|
||||
}
|
||||
|
||||
return &Repository{}
|
||||
}
|
||||
85
pkg/repository/rutracker.go
Normal file
85
pkg/repository/rutracker.go
Normal file
@ -0,0 +1,85 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
iface "git.amok.space/yevhen/resource-scraper/types"
|
||||
"git.amok.space/yevhen/resource-scraper/types/table"
|
||||
)
|
||||
|
||||
type Rutracker struct {
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
func NewRutracker(db *sqlx.DB) *Rutracker {
|
||||
return &Rutracker{db: db}
|
||||
}
|
||||
|
||||
func (s *Rutracker) GetTopic(topics []string) error {
|
||||
endpoint := viper.GetString("endpoint")
|
||||
|
||||
for _, t := range topics {
|
||||
topic, err := fetch(fmt.Sprintf(endpoint, t))
|
||||
if err != nil {
|
||||
slog.Error("couldn't parse topic data", "err", err.Error())
|
||||
}
|
||||
|
||||
for i, e := range topic.Entry {
|
||||
var id int
|
||||
var es table.ExternalSources
|
||||
|
||||
u, _ := url.Parse(e.Link.Href)
|
||||
es.Type = "rutracker"
|
||||
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
|
||||
es.Title = e.Title
|
||||
es.TypeSubsectionId, _ = strconv.Atoi(t)
|
||||
es.Releaser = e.Author.Name
|
||||
es.Created, _ = time.Parse(time.RFC3339, e.Updated)
|
||||
created := es.Created.Format(iface.DateTimeFormat)
|
||||
|
||||
query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable)
|
||||
|
||||
row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created)
|
||||
if err = row.Scan(&id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
|
||||
resp, err := http.Get(endpoint)
|
||||
if err != nil {
|
||||
slog.Error("couldn't fetch data", endpoint, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func(Body io.ReadCloser) {
|
||||
err = Body.Close()
|
||||
if err != nil {
|
||||
fmt.Println("Body.Close")
|
||||
}
|
||||
}(resp.Body)
|
||||
|
||||
topic := &iface.RutrackerAtomTopic{}
|
||||
|
||||
if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return topic, nil
|
||||
}
|
||||
17
pkg/service/rutracker.go
Normal file
17
pkg/service/rutracker.go
Normal file
@ -0,0 +1,17 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
iface "git.amok.space/yevhen/resource-scraper/types"
|
||||
)
|
||||
|
||||
type RutrackerService struct {
|
||||
repo iface.Rutracker
|
||||
}
|
||||
|
||||
func NewRutrackerService(repo iface.Rutracker) *RutrackerService {
|
||||
return &RutrackerService{repo: repo}
|
||||
}
|
||||
|
||||
func (s *RutrackerService) GetTopic(topic []string) error {
|
||||
return s.repo.GetTopic(topic)
|
||||
}
|
||||
24
pkg/service/service.go
Normal file
24
pkg/service/service.go
Normal file
@ -0,0 +1,24 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.amok.space/yevhen/resource-scraper/pkg/repository"
|
||||
"git.amok.space/yevhen/resource-scraper/types"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
types.Rutracker
|
||||
}
|
||||
|
||||
func New(repos *repository.Repository) *Service {
|
||||
|
||||
switch viper.GetString("scope") {
|
||||
case types.RuTracker:
|
||||
return &Service{
|
||||
Rutracker: NewRutrackerService(repos.Rutracker),
|
||||
}
|
||||
}
|
||||
|
||||
return &Service{}
|
||||
}
|
||||
14
types/constants.go
Normal file
14
types/constants.go
Normal file
@ -0,0 +1,14 @@
|
||||
package types
|
||||
|
||||
const (
|
||||
RuTracker string = "rutracker"
|
||||
)
|
||||
|
||||
const (
|
||||
RoleConsole string = "console"
|
||||
)
|
||||
|
||||
const (
|
||||
DateTimeFormat string = "2006-01-02 15:04:05"
|
||||
ExternalSourcesTable string = "external_sources"
|
||||
)
|
||||
5
types/interfaces.go
Normal file
5
types/interfaces.go
Normal file
@ -0,0 +1,5 @@
|
||||
package types
|
||||
|
||||
type Rutracker interface {
|
||||
GetTopic(topics []string) error
|
||||
}
|
||||
35
types/rutracker.go
Normal file
35
types/rutracker.go
Normal file
@ -0,0 +1,35 @@
|
||||
package types
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
type RutrackerAtomTopic struct {
|
||||
XMLName xml.Name `xml:"feed"`
|
||||
Text string `xml:",chardata"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
ID string `xml:"id"`
|
||||
Link struct {
|
||||
Text string `xml:",chardata"`
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"link"`
|
||||
Updated string `xml:"updated"`
|
||||
Title string `xml:"title"`
|
||||
Entry []struct {
|
||||
Text string `xml:",chardata"`
|
||||
ID string `xml:"id"`
|
||||
Link struct {
|
||||
Text string `xml:",chardata"`
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"link"`
|
||||
Updated string `xml:"updated"`
|
||||
Title string `xml:"title"`
|
||||
Author struct {
|
||||
Text string `xml:",chardata"`
|
||||
Name string `xml:"name"`
|
||||
} `xml:"author"`
|
||||
Category struct {
|
||||
Text string `xml:",chardata"`
|
||||
Term string `xml:"term,attr"`
|
||||
Label string `xml:"label,attr"`
|
||||
} `xml:"category"`
|
||||
} `xml:"entry"`
|
||||
}
|
||||
39
types/table/external_sources.go
Normal file
39
types/table/external_sources.go
Normal file
@ -0,0 +1,39 @@
|
||||
package table
|
||||
|
||||
import "time"
|
||||
|
||||
type Type string
|
||||
|
||||
/*const (
|
||||
mmt Type = iota
|
||||
prescene
|
||||
rutracker
|
||||
locate
|
||||
darkabyss
|
||||
bfm
|
||||
trash
|
||||
nnmc
|
||||
stb
|
||||
bandcamp
|
||||
deathgrind
|
||||
)*/
|
||||
|
||||
type ExternalSources struct {
|
||||
Id int `json:"id" db:"id"`
|
||||
Type string `json:"type" db:"type"`
|
||||
TypeId int `json:"type_id" db:"type_id"`
|
||||
Title string `json:"title" db:"title"`
|
||||
TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
|
||||
Releaser string `json:"releaser" db:"releaser"`
|
||||
ExSource string `json:"ex_source" db:"eXsource"`
|
||||
Created time.Time `json:"created_at" db:"created"`
|
||||
Modified time.Time `json:"modified_at" db:"modified"`
|
||||
Contents string `json:"contents" db:"contents"`
|
||||
A string `json:"a" db:"a"`
|
||||
H string `json:"h" db:"h"`
|
||||
Fingerprint string `json:"fingerprint" db:"fingerprint"`
|
||||
FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
|
||||
Vid int `json:"vid" db:"vid"`
|
||||
G string `json:"g" db:"g"`
|
||||
ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user