1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"strings"
"golang.org/x/net/html"
)
type PropertyField struct {
Title string `json:"title"`
Value string `json:"value"`
}
type PropertySection struct {
Section string `json:"section"`
Fields []PropertyField `json:"fields"`
}
func test() {
// Read the HTML file
htmlContent, err := os.ReadFile("oikotie.html")
if err != nil {
log.Fatal("Error reading file:", err)
}
// Parse the HTML
sections, err := parsePropertyHTML(string(htmlContent))
if err != nil {
log.Fatal("Error parsing HTML:", err)
}
// Convert to JSON
jsonData, err := json.MarshalIndent(sections, "", " ")
if err != nil {
log.Fatal("Error marshaling JSON:", err)
}
// Write to file
err = os.WriteFile("property_data.json", jsonData, 0644)
if err != nil {
log.Fatal("Error writing JSON file:", err)
}
fmt.Println("Successfully parsed property data and saved to property_data.json")
}
func parsePropertyHTML(htmlContent string) ([]PropertySection, error) {
doc, err := html.Parse(strings.NewReader(htmlContent))
if err != nil {
return nil, err
}
var sections []PropertySection
var currentSection *PropertySection
// Recursive function to traverse the HTML nodes
var traverse func(*html.Node)
traverse = func(n *html.Node) {
if n.Type == html.ElementNode {
// Check for section headers
if n.Data == "h3" && hasClass(n, "heading") && hasClass(n, "heading--title-2") {
if currentSection != nil && len(currentSection.Fields) > 0 {
sections = append(sections, *currentSection)
}
sectionName := extractText(n)
currentSection = &PropertySection{
Section: sectionName,
Fields: []PropertyField{},
}
}
// Check for info table rows
if n.Data == "div" && hasClass(n, "info-table__row") {
if currentSection != nil {
field := parseInfoTableRow(n)
if field.Title != "" {
currentSection.Fields = append(currentSection.Fields, field)
}
}
}
}
// Traverse child nodes
for c := n.FirstChild; c != nil; c = c.NextSibling {
traverse(c)
}
}
traverse(doc)
// Don't forget to add the last section
if currentSection != nil && len(currentSection.Fields) > 0 {
sections = append(sections, *currentSection)
}
return sections, nil
}
func parseInfoTableRow(n *html.Node) PropertyField {
var field PropertyField
var traverseRow func(*html.Node)
traverseRow = func(node *html.Node) {
if node.Type == html.ElementNode {
if node.Data == "dt" && hasClass(node, "info-table__title") {
field.Title = extractText(node)
}
if node.Data == "dd" && hasClass(node, "info-table__value") {
field.Value = extractText(node)
}
}
for c := node.FirstChild; c != nil; c = c.NextSibling {
traverseRow(c)
}
}
traverseRow(n)
return field
}
func hasClass(n *html.Node, className string) bool {
for _, attr := range n.Attr {
if attr.Key == "class" && strings.Contains(attr.Val, className) {
return true
}
}
return false
}
func extractText(n *html.Node) string {
var text strings.Builder
var extract func(*html.Node)
extract = func(node *html.Node) {
if node.Type == html.TextNode {
text.WriteString(node.Data)
}
for c := node.FirstChild; c != nil; c = c.NextSibling {
extract(c)
}
}
extract(n)
return strings.TrimSpace(text.String())
}
|
