import crypto from "crypto"; import fs from "fs"; import path from "path"; const couchUsername = process.env.COUCHDB_USERNAME; const couchPassword = process.env.COUCHDB_PASSWORD; function getAuthHeader() { if (!couchUsername || !couchPassword) { throw new Error("CouchDB credentials not set in environment variables"); } const auth = Buffer.from(`${couchUsername}:${couchPassword}`).toString("base64"); return `Basic ${auth}`; } // === CONFIG === const baseUrl = "https://kartta.hel.fi/ws/geoserver/avoindata/wfs"; const layers = [ "Aluesarjat_avainluvut_2024", "Piirijako_pienalue", "Piirijako_peruspiiri", "Seutukartta_liikenne_paatiet", "Seutukartta_liikenne_metroasemat", "Seutukartta_liikenne_metro_rata", "Seutukartta_liikenne_juna_rata", "Seutukartta_liikenne_juna_asema", "Seutukartta_aluejako_pienalue", "Seutukartta_aluejako_kuntarajat", "Seutukartta_maankaytto_jarvet", "Seutukartta_maankaytto_joet", "Seutukartta_meren_rantaviiva", "Toimipisterekisteri_palvelut", ]; const outputDir = path.join(process.cwd(), "app", "data"); const couchUrl = "https://couch.tammi.cc"; const dbName = "helsinki_wfs"; // Ensure output dir if (!fs.existsSync(outputDir)) { fs.mkdirSync(outputDir, { recursive: true }); } function getHeaders() { return new Headers({ // biome-ignore lint/style/useNamingConvention: database Authorization: getAuthHeader(), "Content-Type": "application/json", }); } // === COUCHDB HELPERS === async function createDatabase() { const url = `${couchUrl}/${dbName}`; const res = await fetch(url, { headers: getHeaders(), method: "PUT", }); if (res.ok || res.status === 412) { console.log(`Database ${dbName} ready.`); } else { throw new Error(await res.text()); } } async function ensureDesignDocs() { const designDoc = { _id: "_design/layers", views: { // biome-ignore lint/style/useNamingConvention: database by_layer: { map: `function(doc) { if (doc.type === 'feature' && doc.layer) { emit(doc.layer, null); } }`, }, }, }; const url = `${couchUrl}/${dbName}/_design/layers`; const res = await fetch(url, { headers: getHeaders() }); if (res.status === 404) { await fetch(url, { body: JSON.stringify(designDoc), headers: getHeaders(), method: "PUT", }); console.log("Created design document: layers/by_layer"); } else if (res.ok) { const existing = await res.json(); designDoc._rev = existing._rev; await fetch(url, { body: JSON.stringify(designDoc), headers: getHeaders(), method: "PUT", }); console.log("Updated design document"); } } // === DOWNLOAD === async function downloadLayer(layer) { const url = `${baseUrl}?service=WFS&version=2.0.0&request=GetFeature&typeName=avoindata:${layer}&outputFormat=json&srsname=EPSG:4326`; const res = await fetch(url); if (!res.ok) throw new Error(res.statusText); const response = await res.json(); return response; } function saveToFile(layer, data) { const filePath = path.join(outputDir, `${layer}.geojson`); fs.writeFileSync(filePath, JSON.stringify(data, null, "\t")); console.log(`Saved: ${layer}.geojson`); } // === UPLOAD METADATA === async function uploadLayerMetadata(layer, featureCount) { const docId = `layer_metadata:${layer}`; const doc = { _id: docId, // biome-ignore lint/style/useNamingConvention: database feature_count: featureCount, // biome-ignore lint/style/useNamingConvention: database last_updated: new Date().toISOString(), name: layer, projection: "EPSG:4326", type: "layer_metadata", }; const url = `${couchUrl}/${dbName}/${docId}`; const getRes = await fetch(url, { headers: getHeaders() }); if (getRes.ok) { const existing = await getRes.json(); doc._rev = existing._rev; } const putRes = await fetch(url, { body: JSON.stringify(doc), headers: getHeaders(), method: "PUT", }); if (!putRes.ok) throw new Error(await putRes.text()); console.log(`Metadata updated: ${layer} (${featureCount} features)`); } // === UPLOAD SINGLE FEATURE (with deduplication) === async function uploadFeature(doc) { const url = `${couchUrl}/${dbName}/${doc._id}`; const getRes = await fetch(url, { headers: getHeaders() }); if (getRes.ok) { const existing = await getRes.json(); doc._rev = existing._rev; const geomEqual = JSON.stringify(doc.geometry) === JSON.stringify(existing.geometry); const propEqual = JSON.stringify(doc.properties) === JSON.stringify(existing.properties); if (geomEqual && propEqual) { return false; // skipped } } const putRes = await fetch(url, { body: JSON.stringify(doc), headers: getHeaders(), method: "PUT", }); return putRes.ok; } // === PROCESS LAYER === async function processLayer(layer) { const geojson = await downloadLayer(layer); if (!geojson || !geojson.features) { console.warn(`No features in ${layer} ${geojson}`); process.exit(1); } let uploaded = 0; let skipped = 0; for (const feature of geojson.features) { // Stable ID: use feature.id, or property, or UUID const propId = feature.id || feature.properties?.id || feature.properties?.tunnus || feature.properties?.objectid || crypto.randomUUID(); const doc = { _id: `feature:${layer}:${propId}`, // biome-ignore lint/style/useNamingConvention: database downloaded_at: new Date().toISOString(), geometry: feature.geometry, layer: layer, properties: feature.properties || {}, type: "feature", }; const success = await uploadFeature(doc); success ? uploaded++ : skipped++; } await uploadLayerMetadata(layer, geojson.features.length); console.log(`Done: ${layer} | Uploaded: ${uploaded} | Skipped: ${skipped}`); } // === MAIN === async function main() { await createDatabase(); await ensureDesignDocs(); for (const layer of layers) { await processLayer(layer); // Optional: rate limiting await new Promise((r) => setTimeout(r, 500)); } console.log("All layers processed."); } if (process.argv[1] === new URL(import.meta.url).pathname) { main().catch(console.error); }