diff --git a/.air.toml b/.air.toml index 48c91f9..0941617 100644 --- a/.air.toml +++ b/.air.toml @@ -4,7 +4,7 @@ tmp_dir = "tmp" [build] args_bin = [] - bin = "tmp\\main.exe" + bin = "tmp\\main.exe --config-file ./config/default --scope rutracker" cmd = "go build -o ./tmp/main.exe ./cmd" delay = 1000 exclude_dir = ["assets", "tmp", "vendor", "testdata"] diff --git a/.gitignore b/.gitignore index 20a7b3d..b87b2a5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ fabric.properties .idea/httpRequests .idea/caches/build_file_checksums.ser .env -tmp/ \ No newline at end of file +tmp/ +vendor/ \ No newline at end of file diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 0000000..30ae2dc --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,18 @@ + + + + + mariadb + true + org.mariadb.jdbc.Driver + jdbc:mariadb://10.14.88.1:6033/go_tut_tokill + + + + + + + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..ddc2326 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index 5dbf600..e136c02 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,28 +1,16 @@ package main import ( - "fmt" - "log" - "os" + _ "github.com/go-sql-driver/mysql" - "github.com/joho/godotenv" + scraper "git.amok.space/yevhen/resource-scraper/internal" ) -// init is invoked before main() func init() { - // loads values from .env into the system - if err := godotenv.Load(); err != nil { - log.Print("No .env file found") - } + scraper.ParseFlags() } func main() { - // Get the GREETING environment variable - greeting, exists := os.LookupEnv("GREETING") - - if exists { - fmt.Println(greeting) - } - - fmt.Println("fin") + scraper.Bootstrap() + //fmt.Printf("%v\n", viper.Get("23").([]interface{})) } diff --git a/config/default.yaml b/config/default.yaml new file mode 100644 index 0000000..b3f2d12 --- /dev/null +++ b/config/default.yaml @@ -0,0 +1,5 @@ +db: + driver: mysql +# data_source_name: who:is@(10.14.88.1:6033)/go_tut_tokill + data_source_name: theamok_mdb:2ilt3l72(!)r71a0a0d@(10.14.88.1:6033)/theamok_mdb + diff --git a/config/rutracker.yaml b/config/rutracker.yaml new file mode 100644 index 0000000..2415c22 --- /dev/null +++ b/config/rutracker.yaml @@ -0,0 +1,27 @@ +role: console +endpoint: https://feed.rutracker.cc/atom/f/%s.atom +topic: + 0: [ 1719, 1779 ] + 1: [ 1796, 1730 ] + 2: [ 1720, 1728 ] + 3: [ 1724, 1815 ] + 4: [ 1726, 1719 ] + 5: [ 2230, 1779 ] + 6: [ 1868, 1796 ] + 7: [ 1877, 1730 ] + 8: [ 1866, 1720 ] + 9: [ 1864, 1728 ] + 10: [ 1871, 1724 ] + 11: [ 1869, 1815 ] + 12: [ 1788, 1719 ] + 13: [ 739, 1779 ] + 14: [ 951, 1796 ] + 15: [ 1740, 1730 ] + 16: [ 1736, 1720 ] + 17: [ 1746, 1728 ] + 18: [ 1744, 1724 ] + 19: [ 1748, 1815 ] + 20: [ 1706, 1719 ] + 21: [ 1757, 1779 ] + 22: [ 1133, 739 ] + 23: [ 1755, 1756 ] diff --git a/go.mod b/go.mod index 9be0bd9..862db12 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,33 @@ module git.amok.space/yevhen/resource-scraper go 1.23.1 -require github.com/joho/godotenv v1.5.1 // indirect +require ( + github.com/go-chi/chi/v5 v5.1.0 + github.com/go-chi/cors v1.2.1 + github.com/go-sql-driver/mysql v1.8.1 + github.com/jmoiron/sqlx v1.4.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.19.0 +) + +require ( + filippo.io/edwards25519 v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index d61b19e..e736876 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,85 @@ -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= +github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helper/web/handlerReadiness.go b/helper/web/handlerReadiness.go new file mode 100644 index 0000000..89aa7af --- /dev/null +++ b/helper/web/handlerReadiness.go @@ -0,0 +1,7 @@ +package web + +import "net/http" + +func FallbackHandler(w http.ResponseWriter, r *http.Request) { + respondWithJSON(w, http.StatusOK, struct{}{}) +} diff --git a/helper/web/json.go b/helper/web/json.go new file mode 100644 index 0000000..ee2ae10 --- /dev/null +++ b/helper/web/json.go @@ -0,0 +1,35 @@ +package web + +import ( + "encoding/json" + "log" + "net/http" +) + +type errorResponse struct { + Error string `json:"error"` +} + +func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) { + data, err := json.Marshal(payload) + if err != nil { + log.Printf("failed to marshal JSON response: %v\n", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(code) + _, err = w.Write(data) + if err != nil { + log.Printf("failed to write data to output buffer: %v\n", err) + } +} + +func respondWithError(w http.ResponseWriter, code int, message string) { + if code >= http.StatusInternalServerError { + log.Println("responding with 5xx error:", message) + } + + respondWithJSON(w, code, errorResponse{Error: message}) +} diff --git a/internal/bootstrap.go b/internal/bootstrap.go new file mode 100644 index 0000000..2299c04 --- /dev/null +++ b/internal/bootstrap.go @@ -0,0 +1,31 @@ +package scraper + +import ( + "fmt" + + "github.com/spf13/viper" + + "git.amok.space/yevhen/resource-scraper/internal/config" + "git.amok.space/yevhen/resource-scraper/internal/db" + "git.amok.space/yevhen/resource-scraper/pkg/handler" + "git.amok.space/yevhen/resource-scraper/pkg/repository" + "git.amok.space/yevhen/resource-scraper/pkg/service" + "git.amok.space/yevhen/resource-scraper/types" +) + +func Bootstrap() { + config.New() + + dbase := db.New() + repos := repository.New(dbase) + services := service.New(repos) + handlers := handler.New(services) + + switch viper.GetString("role") { + case types.RoleConsole: + fmt.Printf("init console console: %s\n", handlers.InitConsole()) + break + } + + ///http.Run() +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..4609319 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,34 @@ +package config + +import ( + "fmt" + "log" + "path/filepath" + + "github.com/spf13/viper" +) + +func New() { + configFilePath := viper.GetString("config-file") + configDir := "./" + filepath.Dir(configFilePath) + + viper.SetConfigName(filepath.Base(configFilePath)) + viper.AddConfigPath(configDir) + + err := viper.ReadInConfig() // Find and read the config file + if err != nil { // Handle errors reading the config file + panic(fmt.Errorf("fatal error config file: %w", err)) + } + + viper.SetDefault("ConfigDir", configDir) + scope := viper.GetString("scope") + if scope != "" { + viper.SetConfigName(scope) + viper.AddConfigPath(configDir) + err := viper.MergeInConfig() + if err != nil { + log.Fatalf("fatal error config file: %v", err) + return + } + } +} diff --git a/internal/db/connector.go b/internal/db/connector.go new file mode 100644 index 0000000..cd92865 --- /dev/null +++ b/internal/db/connector.go @@ -0,0 +1,24 @@ +package db + +import ( + "github.com/jmoiron/sqlx" + "github.com/spf13/viper" +) + +func New() *sqlx.DB { + drv := viper.GetString("db.driver") + dsn := viper.GetString("db.data_source_name") + + db, err := sqlx.Connect(drv, dsn) + if err != nil { + panic("Failed to connect to the database: " + err.Error()) + } + + // Verify the connection to the database is still alive + err = db.Ping() + if err != nil { + panic("Failed to ping the database: " + err.Error()) + } + + return db +} diff --git a/internal/flag.go b/internal/flag.go new file mode 100644 index 0000000..f7245a0 --- /dev/null +++ b/internal/flag.go @@ -0,0 +1,55 @@ +package scraper + +import ( + "flag" + "fmt" + "log" + "runtime" + + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const usage = `Music Database (MDB) server/cli craftware' + +Usage: + mdb [command] -c [config-file-path] + +Commands: + -help, -h Print this message + -version -v Print version. + -config -c Set configuration file without extension. It is %s by default. + -section -s Select section e.g.: rutracker, bandcamp etc. + +The Go runtime version: %s +Report bugs to https://git.amok.space/yevhen/resource-scraper/issues` + +const ( + maxPasswordLength = 20 + version = "0.1" + defaultConfigPath = "./config/default" +) + +func ParseFlags() { + flag.Bool("h", false, "") + flag.Bool("help", false, "") + flag.Bool("v", false, "") + flag.Bool("version", false, "") + flag.Bool("debug", false, "") + flag.String("config-file", defaultConfigPath, "config file location used for the program") + flag.String("scope", "", "") + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + err := viper.BindPFlags(pflag.CommandLine) + if err != nil { + log.Fatalf("[ERR] Failed to bind flags to config: %s", err) + } + + if viper.GetBool("h") || viper.GetBool("help") { + msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version()) + fmt.Println(msg) + } else if viper.GetBool("v") || viper.GetBool("version") { + fmt.Println("MDB version", version) + } +} diff --git a/internal/http/http.go b/internal/http/http.go new file mode 100644 index 0000000..e03785b --- /dev/null +++ b/internal/http/http.go @@ -0,0 +1,54 @@ +package http + +import ( + "log" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/cors" + + "git.amok.space/yevhen/resource-scraper/helper/web" +) + +/*type Server struct { + router *chi.Mux +}*/ + +/*type Routes struct { + Api *chi.Mux +}*/ + +/*type Handler func(w http.ResponseWriter, r *http.Request) error + +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if err := h(w, r); err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + + _, err := w.Write([]byte("bad")) + if err != nil { + log.Fatalf("Error starting HTTP server: %v", err) + } + } +}*/ + +func Run() { + port := "19576" + + r := chi.NewRouter() + r.Use(cors.Handler(cors.Options{ + AllowedOrigins: []string{"https://*", "http://*"}, + AllowedMethods: []string{"GET", "POST"}, + AllowedHeaders: []string{"*"}, + ExposedHeaders: []string{"Link"}, + AllowCredentials: false, + MaxAge: 300, + })) + + r.Get("/", web.FallbackHandler) + + log.Printf("Server starting on port %v", port) + err := http.ListenAndServe("localhost:"+port, r) + if err != nil { + log.Fatalf("Error starting HTTP server: %v", err) + } +} diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go new file mode 100644 index 0000000..7ca366b --- /dev/null +++ b/pkg/handler/handler.go @@ -0,0 +1,28 @@ +package handler + +import ( + "git.amok.space/yevhen/resource-scraper/pkg/service" +) + +type Handler struct { + services *service.Service +} + +func New(services *service.Service) *Handler { + return &Handler{services: services} +} + +func (h *Handler) InitConsole() string { + return h.rutracker() +} + +//func (h *Handler) Base(services *service.Service) *Handler { +// return &Handler{services: services} +//} + +//func (h *Handler) InitApi() *chi.Mux { +// api := chi.NewRouter() +// api.Get("/", web.ApiFallbackHandler) +// +// return api +//} diff --git a/pkg/handler/rutracker.go b/pkg/handler/rutracker.go new file mode 100644 index 0000000..7729bb2 --- /dev/null +++ b/pkg/handler/rutracker.go @@ -0,0 +1,21 @@ +package handler + +import ( + "fmt" + "log/slog" + "time" + + "github.com/spf13/viper" +) + +func (h *Handler) rutracker() string { + key := fmt.Sprintf("topic.%v", time.Now().Hour()) + topics := viper.GetStringSlice(key) + + err := h.services.Rutracker.GetTopic(topics) + if err != nil { + slog.Error("error occurred while getting topic: ", err.Error()) + } + + return "rt" +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go new file mode 100644 index 0000000..94cb896 --- /dev/null +++ b/pkg/repository/repository.go @@ -0,0 +1,23 @@ +package repository + +import ( + "github.com/jmoiron/sqlx" + "github.com/spf13/viper" + + "git.amok.space/yevhen/resource-scraper/types" +) + +type Repository struct { + types.Rutracker +} + +func New(db *sqlx.DB) *Repository { + switch viper.GetString("scope") { + case types.RuTracker: + return &Repository{ + Rutracker: NewRutracker(db), + } + } + + return &Repository{} +} diff --git a/pkg/repository/rutracker.go b/pkg/repository/rutracker.go new file mode 100644 index 0000000..cf9b0f6 --- /dev/null +++ b/pkg/repository/rutracker.go @@ -0,0 +1,85 @@ +package repository + +import ( + "encoding/xml" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/jmoiron/sqlx" + "github.com/spf13/viper" + + iface "git.amok.space/yevhen/resource-scraper/types" + "git.amok.space/yevhen/resource-scraper/types/table" +) + +type Rutracker struct { + db *sqlx.DB +} + +func NewRutracker(db *sqlx.DB) *Rutracker { + return &Rutracker{db: db} +} + +func (s *Rutracker) GetTopic(topics []string) error { + endpoint := viper.GetString("endpoint") + + for _, t := range topics { + topic, err := fetch(fmt.Sprintf(endpoint, t)) + if err != nil { + slog.Error("couldn't parse topic data", "err", err.Error()) + } + + for i, e := range topic.Entry { + var id int + var es table.ExternalSources + + u, _ := url.Parse(e.Link.Href) + es.Type = "rutracker" + es.TypeId, _ = strconv.Atoi(u.Query().Get("t")) + es.Title = e.Title + es.TypeSubsectionId, _ = strconv.Atoi(t) + es.Releaser = e.Author.Name + es.Created, _ = time.Parse(time.RFC3339, e.Updated) + created := es.Created.Format(iface.DateTimeFormat) + + query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable) + + row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created) + if err = row.Scan(&id); err != nil { + return err + } + + fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>") + } + } + + return nil +} + +func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) { + resp, err := http.Get(endpoint) + if err != nil { + slog.Error("couldn't fetch data", endpoint, err.Error()) + return nil, err + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + fmt.Println("Body.Close") + } + }(resp.Body) + + topic := &iface.RutrackerAtomTopic{} + + if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil { + return nil, err + } + + return topic, nil +} diff --git a/pkg/service/rutracker.go b/pkg/service/rutracker.go new file mode 100644 index 0000000..481f1c8 --- /dev/null +++ b/pkg/service/rutracker.go @@ -0,0 +1,17 @@ +package service + +import ( + iface "git.amok.space/yevhen/resource-scraper/types" +) + +type RutrackerService struct { + repo iface.Rutracker +} + +func NewRutrackerService(repo iface.Rutracker) *RutrackerService { + return &RutrackerService{repo: repo} +} + +func (s *RutrackerService) GetTopic(topic []string) error { + return s.repo.GetTopic(topic) +} diff --git a/pkg/service/service.go b/pkg/service/service.go new file mode 100644 index 0000000..222b9d4 --- /dev/null +++ b/pkg/service/service.go @@ -0,0 +1,24 @@ +package service + +import ( + "github.com/spf13/viper" + + "git.amok.space/yevhen/resource-scraper/pkg/repository" + "git.amok.space/yevhen/resource-scraper/types" +) + +type Service struct { + types.Rutracker +} + +func New(repos *repository.Repository) *Service { + + switch viper.GetString("scope") { + case types.RuTracker: + return &Service{ + Rutracker: NewRutrackerService(repos.Rutracker), + } + } + + return &Service{} +} diff --git a/types/constants.go b/types/constants.go new file mode 100644 index 0000000..7aa479c --- /dev/null +++ b/types/constants.go @@ -0,0 +1,14 @@ +package types + +const ( + RuTracker string = "rutracker" +) + +const ( + RoleConsole string = "console" +) + +const ( + DateTimeFormat string = "2006-01-02 15:04:05" + ExternalSourcesTable string = "external_sources" +) diff --git a/types/interfaces.go b/types/interfaces.go new file mode 100644 index 0000000..d7c7871 --- /dev/null +++ b/types/interfaces.go @@ -0,0 +1,5 @@ +package types + +type Rutracker interface { + GetTopic(topics []string) error +} diff --git a/types/rutracker.go b/types/rutracker.go new file mode 100644 index 0000000..d00633e --- /dev/null +++ b/types/rutracker.go @@ -0,0 +1,35 @@ +package types + +import "encoding/xml" + +type RutrackerAtomTopic struct { + XMLName xml.Name `xml:"feed"` + Text string `xml:",chardata"` + Xmlns string `xml:"xmlns,attr"` + ID string `xml:"id"` + Link struct { + Text string `xml:",chardata"` + Href string `xml:"href,attr"` + } `xml:"link"` + Updated string `xml:"updated"` + Title string `xml:"title"` + Entry []struct { + Text string `xml:",chardata"` + ID string `xml:"id"` + Link struct { + Text string `xml:",chardata"` + Href string `xml:"href,attr"` + } `xml:"link"` + Updated string `xml:"updated"` + Title string `xml:"title"` + Author struct { + Text string `xml:",chardata"` + Name string `xml:"name"` + } `xml:"author"` + Category struct { + Text string `xml:",chardata"` + Term string `xml:"term,attr"` + Label string `xml:"label,attr"` + } `xml:"category"` + } `xml:"entry"` +} diff --git a/types/table/external_sources.go b/types/table/external_sources.go new file mode 100644 index 0000000..66408bc --- /dev/null +++ b/types/table/external_sources.go @@ -0,0 +1,39 @@ +package table + +import "time" + +type Type string + +/*const ( + mmt Type = iota + prescene + rutracker + locate + darkabyss + bfm + trash + nnmc + stb + bandcamp + deathgrind +)*/ + +type ExternalSources struct { + Id int `json:"id" db:"id"` + Type string `json:"type" db:"type"` + TypeId int `json:"type_id" db:"type_id"` + Title string `json:"title" db:"title"` + TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"` + Releaser string `json:"releaser" db:"releaser"` + ExSource string `json:"ex_source" db:"eXsource"` + Created time.Time `json:"created_at" db:"created"` + Modified time.Time `json:"modified_at" db:"modified"` + Contents string `json:"contents" db:"contents"` + A string `json:"a" db:"a"` + H string `json:"h" db:"h"` + Fingerprint string `json:"fingerprint" db:"fingerprint"` + FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"` + Vid int `json:"vid" db:"vid"` + G string `json:"g" db:"g"` + ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"` +}