diff options
| author | Petri Hienonen <petri.hienonen@gmail.com> | 2025-12-28 13:51:36 +0200 |
|---|---|---|
| committer | Petri Hienonen <petri.hienonen@gmail.com> | 2025-12-28 13:51:36 +0200 |
| commit | 341b9d2f8be11d6149994a89ddbde505a0b7977e (patch) | |
| tree | f13fce913488593298c85822e16a8cfdcae90f1c | |
| parent | 0bdadb8f490dbc982954cda7f2c70eec365e05b8 (diff) | |
| download | housing-341b9d2f8be11d6149994a89ddbde505a0b7977e.tar.zst | |
Update golang dependencies
| -rw-r--r-- | flake.nix | 2 | ||||
| -rw-r--r-- | scrape/go.mod | 20 | ||||
| -rw-r--r-- | scrape/go.sum | 41 | ||||
| -rw-r--r-- | scrape/main.go | 52 |
4 files changed, 58 insertions, 57 deletions
@@ -1,5 +1,5 @@ { - description = "Vertical"; + description = "Housing"; inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; }; diff --git a/scrape/go.mod b/scrape/go.mod index bec6e3f..c45b7d6 100644 --- a/scrape/go.mod +++ b/scrape/go.mod @@ -3,26 +3,26 @@ module tammi.cc/housing go 1.25.2 require ( - github.com/minio/minio-go/v7 v7.0.95 - golang.org/x/net v0.47.0 + github.com/minio/minio-go/v7 v7.0.97 + golang.org/x/net v0.48.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.11 // indirect - github.com/minio/crc64nvme v1.0.2 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/tinylib/msgp v1.3.0 // indirect - golang.org/x/crypto v0.44.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/scrape/go.sum b/scrape/go.sum index dde91de..5ebf812 100644 --- a/scrape/go.sum +++ b/scrape/go.sum @@ -4,21 +4,21 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= -github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= -github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= +github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -27,16 +27,17 @@ github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= -golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= -golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scrape/main.go b/scrape/main.go index fed0397..13297bb 100644 --- a/scrape/main.go +++ b/scrape/main.go @@ -76,26 +76,26 @@ func (s *S3Client) UploadFromURL(imgURL, key string) (string, error) { return "", fmt.Errorf("HTTP GET failed: %w", err) } defer resp.Body.Close() - + if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("image download failed with status %d", resp.StatusCode) } - + data, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("reading response body failed: %w", err) } - + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - + _, err = s.client.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ ContentType: "image/webp", }) if err != nil { return "", fmt.Errorf("S3 upload failed: %w", err) } - + publicURL := fmt.Sprintf("https://%s/%s/%s", s.client.EndpointURL().Host, s.bucket, key) log.Printf("Successfully uploaded image: %s", publicURL) return publicURL, nil @@ -133,13 +133,13 @@ func (c *CouchClient) Upsert(h *House) error { if err != nil { return fmt.Errorf("JSON marshal failed: %w", err) } - + reqURL := fmt.Sprintf("%s/%s/%s", c.baseURL, c.database, url.PathEscape(h.ID)) req, err := http.NewRequest("PUT", reqURL, bytes.NewReader(body)) if err != nil { return fmt.Errorf("request creation failed: %w", err) } - + req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") @@ -148,12 +148,12 @@ func (c *CouchClient) Upsert(h *House) error { return fmt.Errorf("HTTP request failed: %w", err) } defer resp.Body.Close() - + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) return fmt.Errorf("couchDB responded with status %d: %s", resp.StatusCode, string(body)) } - + var rev struct { Rev string `json:"rev"` } @@ -193,7 +193,7 @@ func (osi *OikotieScraper) loadTokens() error { if osi.otaToken == "" || osi.otaCuid == "" || osi.otaLoaded == "" || osi.phpSessID == "" { log.Println("Missing one or more tokens – please enter them now:") r := bufio.NewReader(os.Stdin) - + if osi.otaToken == "" { fmt.Print("OTA-token: ") token, err := r.ReadString('\n') @@ -202,7 +202,7 @@ func (osi *OikotieScraper) loadTokens() error { } osi.otaToken = strings.TrimSpace(token) } - + if osi.otaCuid == "" { fmt.Print("OTA-cuid: ") cuid, err := r.ReadString('\n') @@ -211,7 +211,7 @@ func (osi *OikotieScraper) loadTokens() error { } osi.otaCuid = strings.TrimSpace(cuid) } - + if osi.otaLoaded == "" { fmt.Print("OTA-loaded: ") loaded, err := r.ReadString('\n') @@ -220,7 +220,7 @@ func (osi *OikotieScraper) loadTokens() error { } osi.otaLoaded = strings.TrimSpace(loaded) } - + if osi.phpSessID == "" { fmt.Print("PHPSESSID: ") sessID, err := r.ReadString('\n') @@ -264,7 +264,7 @@ func (os *OikotieScraper) ScrapeAll(ctx context.Context, couch *CouchClient) err } continue } - + log.Printf("Fetch error (offset %d): %v", offset, err) retryCount++ if retryCount > maxRetries { @@ -273,7 +273,7 @@ func (os *OikotieScraper) ScrapeAll(ctx context.Context, couch *CouchClient) err time.Sleep(5 * time.Second) continue } - + retryCount = 0 // Reset retry count on successful fetch if len(cards) == 0 { @@ -297,7 +297,7 @@ func (os *OikotieScraper) ScrapeAll(ctx context.Context, couch *CouchClient) err } } - log.Printf("Batch %d-%d: %d/%d cards saved (total: %d, found: %d)", + log.Printf("Batch %d-%d: %d/%d cards saved (total: %d, found: %d)", offset, offset+len(cards)-1, savedInBatch, len(cards), totalSaved, found) if offset+len(cards) >= found { @@ -311,11 +311,11 @@ func (os *OikotieScraper) ScrapeAll(ctx context.Context, couch *CouchClient) err // Fixed: cardId can be number or string, so use json.Number type apiCard struct { - ID json.Number `json:"cardId"` - Type int `json:"cardType"` - SubType int `json:"cardSubType"` - URL string `json:"url"` - Status int `json:"status"` + ID json.Number `json:"cardId"` + Type int `json:"cardType"` + SubType int `json:"cardSubType"` + URL string `json:"url"` + Status int `json:"status"` Data json.RawMessage `json:"data"` Location json.RawMessage `json:"location"` Company json.RawMessage `json:"company"` @@ -367,13 +367,13 @@ func (os *OikotieScraper) fetchPage(offset, limit int) ([]apiCard, int, error) { Found int `json:"found"` Cards []apiCard `json:"cards"` } - + // Read the body first for better error reporting body, err := io.ReadAll(resp.Body) if err != nil { return nil, 0, fmt.Errorf("reading response body failed: %w", err) } - + if err := json.Unmarshal(body, &payload); err != nil { log.Printf("Raw response: %s", string(body)) return nil, 0, fmt.Errorf("JSON unmarshal failed: %w", err) @@ -384,7 +384,7 @@ func (os *OikotieScraper) fetchPage(offset, limit int) ([]apiCard, int, error) { func (os *OikotieScraper) convertCard(c apiCard) (*House, error) { // Convert json.Number to string for the ID cardID := c.ID.String() - + h := &House{ ID: "oikotie_" + cardID, Source: "oikotie", @@ -413,7 +413,7 @@ func (os *OikotieScraper) convertCard(c apiCard) (*House, error) { } h.Images = append(h.Images, publicURL) } - + log.Printf("Successfully converted card %s with %d images", cardID, len(h.Images)) return h, nil } @@ -458,4 +458,4 @@ func main() { log.Fatalf("Scrape failed: %v", err) } log.Println("Scraping completed successfully!") -}
\ No newline at end of file +} |
