add scraping the torrent topics rss files

.air.toml

@@ -4,7 +4,7 @@ tmp_dir = "tmp"
 
 [build]
   args_bin = []
-  bin = "tmp\\main.exe"
+  bin = "tmp\\main.exe --config-file ./config/default --scope rutracker"
   cmd = "go build -o ./tmp/main.exe ./cmd"
   delay = 1000
   exclude_dir = ["assets", "tmp", "vendor", "testdata"]

.gitignore

@@ -29,4 +29,5 @@ fabric.properties
 .idea/httpRequests
 .idea/caches/build_file_checksums.ser
 .env
-tmp/
+tmp/
+vendor/

.idea/dataSources.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="@arm" uuid="e3965414-f0c4-4a05-af48-11e028c626da">
+      <driver-ref>mariadb</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.mariadb.jdbc.Driver</jdbc-driver>
+      <jdbc-url>jdbc:mariadb://10.14.88.1:6033/go_tut_tokill</jdbc-url>
+      <jdbc-additional-properties>
+        <property name="com.intellij.clouds.kubernetes.db.host.port" />
+        <property name="com.intellij.clouds.kubernetes.db.enabled" value="false" />
+        <property name="com.intellij.clouds.kubernetes.db.resource.type" value="Deployment" />
+        <property name="com.intellij.clouds.kubernetes.db.container.port" />
+      </jdbc-additional-properties>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+  </component>
+</project>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,10 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="GoDfaErrorMayBeNotNil" enabled="true" level="WARNING" enabled_by_default="true">
+      <functions>
+        <function importPath="git.amok.space/yevhen/resource-scraper/pkg/repository" name="fetch" />
+      </functions>
+    </inspection_tool>
+  </profile>
+</component>

cmd/main.go

@@ -1,28 +1,16 @@
 package main
 
 import (
-	"fmt"
+	_ "github.com/go-sql-driver/mysql"
-	"log"
-	"os"
 
-	"github.com/joho/godotenv"
+	scraper "git.amok.space/yevhen/resource-scraper/internal"
 )
 
-// init is invoked before main()
 func init() {
-	// loads values from .env into the system
+	scraper.ParseFlags()
-	if err := godotenv.Load(); err != nil {
-		log.Print("No .env file found")
-	}
 }
 
 func main() {
-	// Get the GREETING environment variable
+	scraper.Bootstrap()
-	greeting, exists := os.LookupEnv("GREETING")
+	//fmt.Printf("%v\n", viper.Get("23").([]interface{}))
-
-	if exists {
-		fmt.Println(greeting)
-	}
-
-	fmt.Println("fin")
 }

config/default.yaml

@@ -0,0 +1,5 @@
+db:
+  driver: mysql
+#  data_source_name: who:is@(10.14.88.1:6033)/go_tut_tokill
+  data_source_name: theamok_mdb:2ilt3l72(!)r71a0a0d@(10.14.88.1:6033)/theamok_mdb
+

config/rutracker.yaml

@@ -0,0 +1,27 @@
+role: console
+endpoint: https://feed.rutracker.cc/atom/f/%s.atom
+topic:
+  0: [ 1719, 1779 ]
+  1: [ 1796, 1730 ]
+  2: [ 1720, 1728 ]
+  3: [ 1724, 1815 ]
+  4: [ 1726, 1719 ]
+  5: [ 2230, 1779 ]
+  6: [ 1868, 1796 ]
+  7: [ 1877, 1730 ]
+  8: [ 1866, 1720 ]
+  9: [ 1864, 1728 ]
+  10: [ 1871, 1724 ]
+  11: [ 1869, 1815 ]
+  12: [ 1788, 1719 ]
+  13: [ 739, 1779 ]
+  14: [ 951, 1796 ]
+  15: [ 1740, 1730 ]
+  16: [ 1736, 1720 ]
+  17: [ 1746, 1728 ]
+  18: [ 1744, 1724 ]
+  19: [ 1748, 1815 ]
+  20: [ 1706, 1719 ]
+  21: [ 1757, 1779 ]
+  22: [ 1133, 739 ]
+  23: [ 1755, 1756 ]

go.mod

@@ -2,4 +2,33 @@ module git.amok.space/yevhen/resource-scraper
 
 go 1.23.1
 
-require github.com/joho/godotenv v1.5.1 // indirect
+require (
+	github.com/go-chi/chi/v5 v5.1.0
+	github.com/go-chi/cors v1.2.1
+	github.com/go-sql-driver/mysql v1.8.1
+	github.com/jmoiron/sqlx v1.4.0
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.19.0
+)
+
+require (
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -1,2 +1,85 @@
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
-github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
+github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
+github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
+github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
+github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
+github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
+github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
+github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
+github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

helper/web/handlerReadiness.go

@@ -0,0 +1,7 @@
+package web
+
+import "net/http"
+
+func FallbackHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithJSON(w, http.StatusOK, struct{}{})
+}

helper/web/json.go

@@ -0,0 +1,35 @@
+package web
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+)
+
+type errorResponse struct {
+	Error string `json:"error"`
+}
+
+func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) {
+	data, err := json.Marshal(payload)
+	if err != nil {
+		log.Printf("failed to marshal JSON response: %v\n", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Add("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_, err = w.Write(data)
+	if err != nil {
+		log.Printf("failed to write data to output buffer: %v\n", err)
+	}
+}
+
+func respondWithError(w http.ResponseWriter, code int, message string) {
+	if code >= http.StatusInternalServerError {
+		log.Println("responding with 5xx error:", message)
+	}
+
+	respondWithJSON(w, code, errorResponse{Error: message})
+}

internal/bootstrap.go

@@ -0,0 +1,31 @@
+package scraper
+
+import (
+	"fmt"
+
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/internal/config"
+	"git.amok.space/yevhen/resource-scraper/internal/db"
+	"git.amok.space/yevhen/resource-scraper/pkg/handler"
+	"git.amok.space/yevhen/resource-scraper/pkg/repository"
+	"git.amok.space/yevhen/resource-scraper/pkg/service"
+	"git.amok.space/yevhen/resource-scraper/types"
+)
+
+func Bootstrap() {
+	config.New()
+
+	dbase := db.New()
+	repos := repository.New(dbase)
+	services := service.New(repos)
+	handlers := handler.New(services)
+
+	switch viper.GetString("role") {
+	case types.RoleConsole:
+		fmt.Printf("init console console: %s\n", handlers.InitConsole())
+		break
+	}
+
+	///http.Run()
+}

internal/config/config.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"fmt"
+	"log"
+	"path/filepath"
+
+	"github.com/spf13/viper"
+)
+
+func New() {
+	configFilePath := viper.GetString("config-file")
+	configDir := "./" + filepath.Dir(configFilePath)
+
+	viper.SetConfigName(filepath.Base(configFilePath))
+	viper.AddConfigPath(configDir)
+
+	err := viper.ReadInConfig() // Find and read the config file
+	if err != nil {             // Handle errors reading the config file
+		panic(fmt.Errorf("fatal error config file: %w", err))
+	}
+
+	viper.SetDefault("ConfigDir", configDir)
+	scope := viper.GetString("scope")
+	if scope != "" {
+		viper.SetConfigName(scope)
+		viper.AddConfigPath(configDir)
+		err := viper.MergeInConfig()
+		if err != nil {
+			log.Fatalf("fatal error config file: %v", err)
+			return
+		}
+	}
+}

internal/db/connector.go

@@ -0,0 +1,24 @@
+package db
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+)
+
+func New() *sqlx.DB {
+	drv := viper.GetString("db.driver")
+	dsn := viper.GetString("db.data_source_name")
+
+	db, err := sqlx.Connect(drv, dsn)
+	if err != nil {
+		panic("Failed to connect to the database: " + err.Error())
+	}
+
+	// Verify the connection to the database is still alive
+	err = db.Ping()
+	if err != nil {
+		panic("Failed to ping the database: " + err.Error())
+	}
+
+	return db
+}

internal/flag.go

@@ -0,0 +1,55 @@
+package scraper
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"runtime"
+
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+)
+
+const usage = `Music Database (MDB) server/cli craftware'
+
+Usage:
+	mdb [command] -c [config-file-path]
+
+Commands:
+	-help,		-h  Print this message
+	-version	-v  Print version.
+	-config		-c  Set configuration file without extension. It is %s by default.
+	-section	-s  Select section e.g.: rutracker, bandcamp etc.
+
+The Go runtime version: %s
+Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
+
+const (
+	maxPasswordLength = 20
+	version           = "0.1"
+	defaultConfigPath = "./config/default"
+)
+
+func ParseFlags() {
+	flag.Bool("h", false, "")
+	flag.Bool("help", false, "")
+	flag.Bool("v", false, "")
+	flag.Bool("version", false, "")
+	flag.Bool("debug", false, "")
+	flag.String("config-file", defaultConfigPath, "config file location used for the program")
+	flag.String("scope", "", "")
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
+	err := viper.BindPFlags(pflag.CommandLine)
+	if err != nil {
+		log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
+	}
+
+	if viper.GetBool("h") || viper.GetBool("help") {
+		msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
+		fmt.Println(msg)
+	} else if viper.GetBool("v") || viper.GetBool("version") {
+		fmt.Println("MDB version", version)
+	}
+}

internal/http/http.go

@@ -0,0 +1,54 @@
+package http
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/cors"
+
+	"git.amok.space/yevhen/resource-scraper/helper/web"
+)
+
+/*type Server struct {
+	router *chi.Mux
+}*/
+
+/*type Routes struct {
+	Api *chi.Mux
+}*/
+
+/*type Handler func(w http.ResponseWriter, r *http.Request) error
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if err := h(w, r); err != nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+
+		_, err := w.Write([]byte("bad"))
+		if err != nil {
+			log.Fatalf("Error starting HTTP server: %v", err)
+		}
+	}
+}*/
+
+func Run() {
+	port := "19576"
+
+	r := chi.NewRouter()
+	r.Use(cors.Handler(cors.Options{
+		AllowedOrigins:   []string{"https://*", "http://*"},
+		AllowedMethods:   []string{"GET", "POST"},
+		AllowedHeaders:   []string{"*"},
+		ExposedHeaders:   []string{"Link"},
+		AllowCredentials: false,
+		MaxAge:           300,
+	}))
+
+	r.Get("/", web.FallbackHandler)
+
+	log.Printf("Server starting on port %v", port)
+	err := http.ListenAndServe("localhost:"+port, r)
+	if err != nil {
+		log.Fatalf("Error starting HTTP server: %v", err)
+	}
+}

pkg/handler/handler.go

@@ -0,0 +1,28 @@
+package handler
+
+import (
+	"git.amok.space/yevhen/resource-scraper/pkg/service"
+)
+
+type Handler struct {
+	services *service.Service
+}
+
+func New(services *service.Service) *Handler {
+	return &Handler{services: services}
+}
+
+func (h *Handler) InitConsole() string {
+	return h.rutracker()
+}
+
+//func (h *Handler) Base(services *service.Service) *Handler {
+//	return &Handler{services: services}
+//}
+
+//func (h *Handler) InitApi() *chi.Mux {
+//	api := chi.NewRouter()
+//	api.Get("/", web.ApiFallbackHandler)
+//
+//	return api
+//}

pkg/handler/rutracker.go

@@ -0,0 +1,21 @@
+package handler
+
+import (
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+func (h *Handler) rutracker() string {
+	key := fmt.Sprintf("topic.%v", time.Now().Hour())
+	topics := viper.GetStringSlice(key)
+
+	err := h.services.Rutracker.GetTopic(topics)
+	if err != nil {
+		slog.Error("error occurred while getting topic: ", err.Error())
+	}
+
+	return "rt"
+}

pkg/repository/repository.go

@@ -0,0 +1,23 @@
+package repository
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/types"
+)
+
+type Repository struct {
+	types.Rutracker
+}
+
+func New(db *sqlx.DB) *Repository {
+	switch viper.GetString("scope") {
+	case types.RuTracker:
+		return &Repository{
+			Rutracker: NewRutracker(db),
+		}
+	}
+
+	return &Repository{}
+}

pkg/repository/rutracker.go

@@ -0,0 +1,85 @@
+package repository
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+
+	iface "git.amok.space/yevhen/resource-scraper/types"
+	"git.amok.space/yevhen/resource-scraper/types/table"
+)
+
+type Rutracker struct {
+	db *sqlx.DB
+}
+
+func NewRutracker(db *sqlx.DB) *Rutracker {
+	return &Rutracker{db: db}
+}
+
+func (s *Rutracker) GetTopic(topics []string) error {
+	endpoint := viper.GetString("endpoint")
+
+	for _, t := range topics {
+		topic, err := fetch(fmt.Sprintf(endpoint, t))
+		if err != nil {
+			slog.Error("couldn't parse topic data", "err", err.Error())
+		}
+
+		for i, e := range topic.Entry {
+			var id int
+			var es table.ExternalSources
+
+			u, _ := url.Parse(e.Link.Href)
+			es.Type = "rutracker"
+			es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
+			es.Title = e.Title
+			es.TypeSubsectionId, _ = strconv.Atoi(t)
+			es.Releaser = e.Author.Name
+			es.Created, _ = time.Parse(time.RFC3339, e.Updated)
+			created := es.Created.Format(iface.DateTimeFormat)
+
+			query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable)
+
+			row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created)
+			if err = row.Scan(&id); err != nil {
+				return err
+			}
+
+			fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>")
+		}
+	}
+
+	return nil
+}
+
+func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
+	resp, err := http.Get(endpoint)
+	if err != nil {
+		slog.Error("couldn't fetch data", endpoint, err.Error())
+		return nil, err
+	}
+
+	defer func(Body io.ReadCloser) {
+		err = Body.Close()
+		if err != nil {
+			fmt.Println("Body.Close")
+		}
+	}(resp.Body)
+
+	topic := &iface.RutrackerAtomTopic{}
+
+	if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
+		return nil, err
+	}
+
+	return topic, nil
+}

pkg/service/rutracker.go

@@ -0,0 +1,17 @@
+package service
+
+import (
+	iface "git.amok.space/yevhen/resource-scraper/types"
+)
+
+type RutrackerService struct {
+	repo iface.Rutracker
+}
+
+func NewRutrackerService(repo iface.Rutracker) *RutrackerService {
+	return &RutrackerService{repo: repo}
+}
+
+func (s *RutrackerService) GetTopic(topic []string) error {
+	return s.repo.GetTopic(topic)
+}

pkg/service/service.go

@@ -0,0 +1,24 @@
+package service
+
+import (
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/pkg/repository"
+	"git.amok.space/yevhen/resource-scraper/types"
+)
+
+type Service struct {
+	types.Rutracker
+}
+
+func New(repos *repository.Repository) *Service {
+
+	switch viper.GetString("scope") {
+	case types.RuTracker:
+		return &Service{
+			Rutracker: NewRutrackerService(repos.Rutracker),
+		}
+	}
+
+	return &Service{}
+}

types/constants.go

@@ -0,0 +1,14 @@
+package types
+
+const (
+	RuTracker string = "rutracker"
+)
+
+const (
+	RoleConsole string = "console"
+)
+
+const (
+	DateTimeFormat       string = "2006-01-02 15:04:05"
+	ExternalSourcesTable string = "external_sources"
+)

types/interfaces.go

@@ -0,0 +1,5 @@
+package types
+
+type Rutracker interface {
+	GetTopic(topics []string) error
+}

types/rutracker.go

@@ -0,0 +1,35 @@
+package types
+
+import "encoding/xml"
+
+type RutrackerAtomTopic struct {
+	XMLName xml.Name `xml:"feed"`
+	Text    string   `xml:",chardata"`
+	Xmlns   string   `xml:"xmlns,attr"`
+	ID      string   `xml:"id"`
+	Link    struct {
+		Text string `xml:",chardata"`
+		Href string `xml:"href,attr"`
+	} `xml:"link"`
+	Updated string `xml:"updated"`
+	Title   string `xml:"title"`
+	Entry   []struct {
+		Text string `xml:",chardata"`
+		ID   string `xml:"id"`
+		Link struct {
+			Text string `xml:",chardata"`
+			Href string `xml:"href,attr"`
+		} `xml:"link"`
+		Updated string `xml:"updated"`
+		Title   string `xml:"title"`
+		Author  struct {
+			Text string `xml:",chardata"`
+			Name string `xml:"name"`
+		} `xml:"author"`
+		Category struct {
+			Text  string `xml:",chardata"`
+			Term  string `xml:"term,attr"`
+			Label string `xml:"label,attr"`
+		} `xml:"category"`
+	} `xml:"entry"`
+}

types/table/external_sources.go

@@ -0,0 +1,39 @@
+package table
+
+import "time"
+
+type Type string
+
+/*const (
+	mmt Type = iota
+	prescene
+	rutracker
+	locate
+	darkabyss
+	bfm
+	trash
+	nnmc
+	stb
+	bandcamp
+	deathgrind
+)*/
+
+type ExternalSources struct {
+	Id                 int       `json:"id" db:"id"`
+	Type               string    `json:"type" db:"type"`
+	TypeId             int       `json:"type_id" db:"type_id"`
+	Title              string    `json:"title" db:"title"`
+	TypeSubsectionId   int       `json:"type_subsection_id" db:"type_subsection_id"`
+	Releaser           string    `json:"releaser" db:"releaser"`
+	ExSource           string    `json:"ex_source" db:"eXsource"`
+	Created            time.Time `json:"created_at" db:"created"`
+	Modified           time.Time `json:"modified_at" db:"modified"`
+	Contents           string    `json:"contents" db:"contents"`
+	A                  string    `json:"a" db:"a"`
+	H                  string    `json:"h" db:"h"`
+	Fingerprint        string    `json:"fingerprint" db:"fingerprint"`
+	FsFingerprint      string    `json:"fs_fingerprint" db:"fs_fingerprint"`
+	Vid                int       `json:"vid" db:"vid"`
+	G                  string    `json:"g" db:"g"`
+	ExternalSourcesCol string    `json:"external_sources_column" db:"external_sourcescol"`
+}