From 5eba467a7eb84409aa43df83de78ecb843a79d7b Mon Sep 17 00:00:00 2001 From: Petri Hienonen Date: Sun, 9 Nov 2025 22:17:04 +0200 Subject: Update --- app/geometry.js | 5 +- app/main.js | 1 - app/models.js | 88 +++++++++++++------ biome.json | 2 +- download.js | 264 +++++++++++++++++++++++++++++++++++++++++++++++--------- jsconfig.json | 2 +- 6 files changed, 289 insertions(+), 73 deletions(-) diff --git a/app/geometry.js b/app/geometry.js index 9b89106..d0f5467 100644 --- a/app/geometry.js +++ b/app/geometry.js @@ -146,7 +146,10 @@ export class Geometry { return Polygon.fromGeoJSON(geojson); case "MultiLineString": return MultiLineString.fromGeoJSON(geojson); + case "MultiPolygon": + return Polygon.fromGeoJSON(geojson); default: + debugger; throw new Error(`Invalid GeoJSON object: missing required 'type' property`); } } @@ -1031,7 +1034,6 @@ export class Feature { * @param {string|number} [id] - Feature ID */ constructor(geometry, properties = {}, id = "") { - this.type = "Feature"; this.geometry = geometry; this.properties = properties; this.id = id; @@ -1070,7 +1072,6 @@ export class Collection { * @param {Feature[]} features - Feature array */ constructor(features = []) { - this.type = "FeatureCollection"; this.features = features; } diff --git a/app/main.js b/app/main.js index 30770ed..1273a11 100644 --- a/app/main.js +++ b/app/main.js @@ -456,7 +456,6 @@ export class App { DataProvider.getCoastline(), DataProvider.getMainRoads(), ]); - this.#districts = districts; this.#houses = houses; this.#trainStations = trainStations; diff --git a/app/models.js b/app/models.js index 2d85df5..9a0e2ca 100644 --- a/app/models.js +++ b/app/models.js @@ -1,4 +1,4 @@ -import { Bounds, Collection, Feature, LineString, Point, Polygon } from "geom"; +import { Bounds, Collection, Feature, Geometry, LineString, Point, Polygon } from "geom"; /** @typedef {{ lat: number, lng: number }} GeoPointJson */ /** @typedef {{ date: string, price: number }} PriceUpdateJson */ @@ -160,10 +160,12 @@ export class Geospatial { export class District { /** * @param {string} name + * @param {string} municipality * @param {Polygon} polygon */ - constructor(name, polygon) { + constructor(name, municipality, polygon) { this.name = name; + this.municipality = municipality; this.polygon = polygon; } @@ -172,14 +174,20 @@ export class District { * @returns {District} */ static fromFeature(feature) { - const name = "nimi_fi" in feature.properties ? feature.properties.nimi_fi : ""; + const name = + "nimi" in feature.properties && typeof feature.properties.nimi === "string" + ? feature.properties.nimi + : ""; + const municipality = + "kunta" in feature.properties && typeof feature.properties.kunta === "string" + ? feature.properties.kunta + : ""; const geometry = feature.geometry; - - if (name === null || name === undefined || !(geometry instanceof Polygon)) { - throw new Error("Invalid district feature data"); + if (!(geometry instanceof Polygon)) { + throw new Error(`Invalid district feature data ${geometry}`); } - return new District(name, geometry); + return new District(name, municipality, geometry); } /** @@ -487,53 +495,77 @@ export class Filters { } export class DataProvider { + static couchBaseUrl = "https://couch.tammi.cc"; + static wfsDbName = "helsinki_wfs"; + static housesDbName = "asunnot"; + + /** + * Fetch all features for a layer as a GeoJSON FeatureCollection + * @param {string} layerName + * @returns {Promise} + */ + static async getCollectionFromCouch(layerName) { + // Use CouchDB view to get all features for the layer + const viewUrl = `${DataProvider.couchBaseUrl}/${DataProvider.wfsDbName}/_design/layers/_view/by_layer`; + const params = new URLSearchParams({ + include_docs: "true", + key: JSON.stringify(layerName), + }); + + const response = await fetch(`${viewUrl}?${params}`, { + headers: new Headers({ accept: "application/json" }), + mode: "cors", + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const result = await response.json(); + const features = result.rows + .map((row) => { + return row.doc.geometry && "type" in row.doc.geometry + ? new Feature(Geometry.fromGeoJSON(row.doc.geometry), row.doc.properties, row.doc._id) + : null; + }) + .filter((x) => x !== null); + return new Collection(features); + } + /** @returns {Promise} */ static async getCoastline() { - return await DataProvider.getCollection("data/Seutukartta_meren_rantaviiva.json"); + return await DataProvider.getCollectionFromCouch("Seutukartta_meren_rantaviiva"); } /** @returns {Promise} */ static async getMainRoads() { - return await DataProvider.getCollection("data/Seutukartta_liikenne_paatiet.json"); + return await DataProvider.getCollectionFromCouch("Seutukartta_liikenne_paatiet"); } /** @returns {Promise} */ static async getDistricts() { - const collection = await DataProvider.getCollection("data/Seutukartta_aluejako_pienalue.json"); - return District.fromCollection(collection); + const collection = await DataProvider.getCollectionFromCouch("Seutukartta_aluejako_pienalue"); + const districts = District.fromCollection(collection); + return districts; } /** @returns {Promise} */ static async getTrainStations() { - const collection = await DataProvider.getCollection( - "data/Seutukartta_liikenne_juna_asema.json", - ); + const collection = await DataProvider.getCollectionFromCouch("Seutukartta_liikenne_juna_asema"); return TrainStation.fromCollection(collection); } /** @returns {Promise} */ static async getTrainTracks() { - const collection = await DataProvider.getCollection("data/Seutukartta_liikenne_juna_rata.json"); + const collection = await DataProvider.getCollectionFromCouch("Seutukartta_liikenne_juna_rata"); return TrainTracks.fromCollection(collection); } - /** - * Load any GeoJSON file as Feature Collection - * @param {string} url - * @returns {Promise} - */ - static async getCollection(url) { - const response = await fetch(url); - if (!response.ok) throw new Error(`Failed to load GeoJSON from ${url}: ${response.status}`); - const geojson = await response.json(); - return Collection.fromGeoJSON(geojson); - } - /** @returns {Promise} */ static async getHouses() { try { const response = await fetch( - new URL("/asunnot/_all_docs?include_docs=true", "https://couch.tammi.cc"), + `${DataProvider.couchBaseUrl}/${DataProvider.housesDbName}/_all_docs?include_docs=true`, { headers: new Headers({ accept: "application/json" }), mode: "cors", diff --git a/biome.json b/biome.json index 675f3fb..bacad07 100644 --- a/biome.json +++ b/biome.json @@ -12,7 +12,7 @@ }, "files": { "ignoreUnknown": true, - "includes": ["./app/*", "./server.js", "./*.json", "./app/data/*.json"], + "includes": ["./app/*", "./*.js", "./*.json"], "maxSize": 10000000 }, "formatter": { diff --git a/download.js b/download.js index d5c4f62..ad149d7 100644 --- a/download.js +++ b/download.js @@ -1,57 +1,241 @@ -const fs = require('fs'); -const path = require('path'); +import crypto from "crypto"; +import fs from "fs"; +import path from "path"; -// Base URL for the WFS service -const baseUrl = 'https://kartta.hel.fi/ws/geoserver/avoindata/wfs'; +const couchUsername = process.env.COUCHDB_USERNAME; +const couchPassword = process.env.COUCHDB_PASSWORD; -// List of layers to download (extendable by adding more items to this array) +function getAuthHeader() { + if (!couchUsername || !couchPassword) { + throw new Error("CouchDB credentials not set in environment variables"); + } + const auth = Buffer.from(`${couchUsername}:${couchPassword}`).toString("base64"); + return `Basic ${auth}`; +} + +// === CONFIG === +const baseUrl = "https://kartta.hel.fi/ws/geoserver/avoindata/wfs"; const layers = [ - 'Aluesarjat_avainluvut_2024', - 'Seutukartta_liikenne_paatiet', - 'Seutukartta_liikenne_metroasemat', - 'Seutukartta_liikenne_metro_rata', - 'Seutukartta_liikenne_juna_rata', - 'Seutukartta_liikenne_juna_asema', - 'Seutukartta_aluejako_pienalue', - 'Seutukartta_aluejako_kuntarajat', - 'Seutukartta_maankaytto_jarvet', - 'Seutukartta_maankaytto_joet', - 'Seutukartta_meren_rantaviiva', - 'Toimipisterekisteri_palvelut' + "Aluesarjat_avainluvut_2024", + "Piirijako_pienalue", + "Seutukartta_liikenne_paatiet", + "Seutukartta_liikenne_metroasemat", + "Seutukartta_liikenne_metro_rata", + "Seutukartta_liikenne_juna_rata", + "Seutukartta_liikenne_juna_asema", + "Seutukartta_aluejako_pienalue", + "Seutukartta_aluejako_kuntarajat", + "Seutukartta_maankaytto_jarvet", + "Seutukartta_maankaytto_joet", + "Seutukartta_meren_rantaviiva", + "Toimipisterekisteri_palvelut", ]; -// Output directory -const outputDir = path.join(__dirname, 'app', 'data'); +const outputDir = path.join(process.cwd(), "app", "data"); +const couchUrl = "https://couch.tammi.cc"; +const dbName = "helsinki_wfs"; -// Create output directory if it doesn't exist +// Ensure output dir if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); + fs.mkdirSync(outputDir, { recursive: true }); +} + +function getHeaders() { + return new Headers({ + Authorization: getAuthHeader(), + "Content-Type": "application/json", + }); +} + +// === COUCHDB HELPERS === +async function createDatabase() { + const url = `${couchUrl}/${dbName}`; + try { + const res = await fetch(url, { + headers: getHeaders(), + method: "PUT", + }); + if (res.ok || res.status === 412) { + console.log(`Database ${dbName} ready.`); + } else { + throw new Error(await res.text()); + } + } catch (e) { + console.error("DB create error:", e.message); + } +} + +async function ensureDesignDocs() { + const designDoc = { + _id: "_design/layers", + views: { + by_layer: { + map: `function(doc) { + if (doc.type === 'feature' && doc.layer) { + emit(doc.layer, null); + } + }`, + }, + }, + }; + + const url = `${couchUrl}/${dbName}/_design/layers`; + try { + const res = await fetch(url, { headers: getHeaders() }); + if (res.status === 404) { + await fetch(url, { + body: JSON.stringify(designDoc), + headers: getHeaders(), + method: "PUT", + }); + console.log("Created design document: layers/by_layer"); + } else if (res.ok) { + const existing = await res.json(); + designDoc._rev = existing._rev; + await fetch(url, { + body: JSON.stringify(designDoc), + headers: getHeaders(), + method: "PUT", + }); + console.log("Updated design document"); + } + } catch (e) { + console.error("Design doc error:", e.message); + process.exit(1); + } } -// Function to download and save layer data +// === DOWNLOAD === async function downloadLayer(layer) { - const url = `${baseUrl}?service=WFS&version=2.0.0&request=GetFeature&typeName=avoindata:${layer}&outputFormat=application/json&srsname=EPSG:4326`; + const url = `${baseUrl}?service=WFS&version=2.0.0&request=GetFeature&typeName=avoindata:${layer}&outputFormat=json&srsname=EPSG:4326`; + try { + const res = await fetch(url); + if (!res.ok) throw new Error(res.statusText); + const response = await res.json(); + return response; + } catch (e) { + console.error(`Download: \n${url}\nfailed [${layer}] ${e.toString()}`); + return null; + } +} - try { - const response = await fetch(url); - if (!response.ok) { - throw new Error(`Failed to fetch ${layer}: ${response.statusText}`); - } - const data = await response.json(); +function saveToFile(layer, data) { + const filePath = path.join(outputDir, `${layer}.geojson`); + fs.writeFileSync(filePath, JSON.stringify(data, null, "\t")); + console.log(`Saved: ${layer}.geojson`); +} - const filePath = path.join(outputDir, `${layer}.json`); - fs.writeFileSync(filePath, JSON.stringify(data, null, '\t')); - console.log(`Downloaded and saved: ${layer}.json`); - } catch (error) { - console.error(`Error downloading ${layer}: ${error.message}`); - } +// === UPLOAD METADATA === +async function uploadLayerMetadata(layer, featureCount) { + const docId = `layer_metadata:${layer}`; + const doc = { + _id: docId, + feature_count: featureCount, + last_updated: new Date().toISOString(), + name: layer, + projection: "EPSG:4326", + type: "layer_metadata", + }; + + const url = `${couchUrl}/${dbName}/${docId}`; + try { + const getRes = await fetch(url, { headers: getHeaders() }); + if (getRes.ok) { + const existing = await getRes.json(); + doc._rev = existing._rev; + } + const putRes = await fetch(url, { + body: JSON.stringify(doc), + headers: getHeaders(), + method: "PUT", + }); + if (!putRes.ok) throw new Error(await putRes.text()); + console.log(`Metadata updated: ${layer} (${featureCount} features)`); + } catch (e) { + console.error(`Metadata error [${layer}]:`, e.message); + } } -// Download all layers sequentially +// === UPLOAD SINGLE FEATURE (with deduplication) === +async function uploadFeature(doc) { + const url = `${couchUrl}/${dbName}/${doc._id}`; + try { + const getRes = await fetch(url, { headers: getHeaders() }); + if (getRes.ok) { + const existing = await getRes.json(); + doc._rev = existing._rev; + + const geomEqual = JSON.stringify(doc.geometry) === JSON.stringify(existing.geometry); + const propEqual = JSON.stringify(doc.properties) === JSON.stringify(existing.properties); + if (geomEqual && propEqual) { + return false; // skipped + } + } + + const putRes = await fetch(url, { + body: JSON.stringify(doc), + headers: getHeaders(), + method: "PUT", + }); + + return putRes.ok; + } catch (e) { + console.warn(`Upload failed [${doc._id}]:`, e.message); + return false; + } +} + +// === PROCESS LAYER === +async function processLayer(layer) { + const geojson = await downloadLayer(layer); + if (!geojson || !geojson.features) { + console.warn(`No features in ${layer} ${geojson}`); + process.exit(1); + } + + let uploaded = 0; + let skipped = 0; + + for (const feature of geojson.features) { + // Stable ID: use feature.id, or property, or UUID + const propId = + feature.id || + feature.properties?.id || + feature.properties?.tunnus || + feature.properties?.objectid || + crypto.randomUUID(); + + const doc = { + _id: `feature:${layer}:${propId}`, + downloaded_at: new Date().toISOString(), + geometry: feature.geometry, + layer: layer, + properties: feature.properties || {}, + type: "feature", + }; + + const success = await uploadFeature(doc); + success ? uploaded++ : skipped++; + } + + await uploadLayerMetadata(layer, geojson.features.length); + console.log(`Done: ${layer} | Uploaded: ${uploaded} | Skipped: ${skipped}`); +} + +// === MAIN === async function main() { - for (const layer of layers) { - await downloadLayer(layer); - } + await createDatabase(); + await ensureDesignDocs(); + + for (const layer of layers) { + await processLayer(layer); + // Optional: rate limiting + await new Promise((r) => setTimeout(r, 500)); + } + + console.log("All layers processed."); } -main(); +if (process.argv[1] === new URL(import.meta.url).pathname) { + main().catch(console.error); +} diff --git a/jsconfig.json b/jsconfig.json index cd0a987..1d6d924 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -28,7 +28,7 @@ "target": "esnext" }, "html.format.enable": false, - "include": ["app/*.js"], + "include": ["app/*.js", "*.js"], "javascript.format.enable": false, "json.format.enable": false, "typescript.format.enable": false -- cgit v1.2.3-70-g09d2