79 lines
2.2 KiB
Go
79 lines
2.2 KiB
Go
package table
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"strings"
|
|
|
|
"github.com/jmoiron/sqlx"
|
|
"github.com/logrusorgru/aurora/v4"
|
|
|
|
"git.amok.space/yevhen/resource-scraper/helper/thither"
|
|
"git.amok.space/yevhen/resource-scraper/types/constant"
|
|
"git.amok.space/yevhen/resource-scraper/types/model"
|
|
)
|
|
|
|
type ExternalSources struct {
|
|
Columns []string
|
|
}
|
|
|
|
func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.DB) model.ExternalSources {
|
|
stmt := "INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE title=:title, created=:created RETURNING id"
|
|
placeholders := strings.Join(f.Columns, ", :")
|
|
placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
|
|
query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
|
|
|
|
if rows, err := db.NamedQuery(query, &es); err == nil {
|
|
for rows.Next() {
|
|
es.Error = rows.StructScan(&es)
|
|
}
|
|
} else {
|
|
es.Error = err
|
|
}
|
|
|
|
return es
|
|
}
|
|
|
|
func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, columns []string) ([]model.ExternalSources, string) {
|
|
es := &ExternalSources{Columns: columns}
|
|
typeIds := es.GetTypeIds(entries, db)
|
|
var status string
|
|
errCount := 0
|
|
|
|
for i := 0; i < len(entries); i++ {
|
|
entry := es.InsertOnDuplicate(entries[i], db)
|
|
if entry.Error != nil {
|
|
slog.Error("insert/update entry", "err", entry.Error)
|
|
errCount++
|
|
}
|
|
|
|
if !slices.Contains(typeIds, entry.TypeId) {
|
|
fmt.Printf("%s: %s\n", aurora.Green("ADDED"), aurora.White(entry.Title))
|
|
}
|
|
|
|
entries[i] = es.InsertOnDuplicate(entries[i], db)
|
|
}
|
|
|
|
if errCount == 0 {
|
|
status = constant.StatusSucceed
|
|
} else if errCount > 0 && errCount == len(entries) {
|
|
status = constant.StatusFailed
|
|
} else {
|
|
status = constant.StatusSuspicious
|
|
}
|
|
|
|
return entries, status
|
|
}
|
|
|
|
func (f *ExternalSources) GetTypeIds(entries []model.ExternalSources, db *sqlx.DB) []int {
|
|
var typeIds []int
|
|
ids := thither.FieldValueToStrSlice(entries, "TypeId")
|
|
query := fmt.Sprintf("SELECT type_id FROM %s WHERE `type` = '%s' AND type_id IN (%s) LIMIT %d", constant.ExternalSourcesTable, entries[0].Type, strings.Join(ids, ","), len(ids))
|
|
err := db.Select(&typeIds, query)
|
|
if err != nil {
|
|
slog.Error("getting type ids", "err", err)
|
|
}
|
|
return typeIds
|
|
}
|