shithub: opossum

Info  •  Files  •  Log

website.go

--- a/browser/browser.go

+++ b/browser/browser.go

@@ -12,13 +12,11 @@

 	"net/http/cookiejar"

 	"net/url"

 	"opossum"

-	"opossum/domino"

 	"opossum/img"

 	"opossum/logger"

 	"opossum/nodes"

 	"opossum/style"

 	"strings"

-	//"unicode"

 	"github.com/mjl-/duit"

@@ -38,7 +36,6 @@

 var browser *Browser // TODO: limit global objects;

 //       at least put them in separate pkgs

 //       with good choiced private/public

-//       p

 var Style = style.Map{}

 var dui *duit.DUI

 var colorCache = make(map[draw.Color]*draw.Image)

@@ -415,62 +412,6 @@

 	n.Attr = append(n.Attr, newAttr)

 }

-func formData(n html.Node) (data url.Values) {

-	data = make(url.Values)

-	if n.Data == "input" {

-		if k := attr(n, "name"); k != "" {

-			data.Set(k, attr(n, "value"))

-		}

-	}

-	for c := n.FirstChild; c != nil; c = c.NextSibling {

-		for k, vs := range formData(*c) {

-			data.Set(k, vs[0]) // TODO: what aboot the rest?

-		}

-	}

-	return

-}

-

-func (b *Browser) submit(form *html.Node) {

-	var err error

-	method := "GET" // TODO

-	if m := attr(*form, "method"); m != "" {

-		method = strings.ToUpper(m)

-	}

-	uri := b.URL()

-	log.Printf("form = %+v", form)

-	if action := attr(*form, "action"); action != "" {

-		uri, err = b.LinkedUrl(action)

-		if err != nil {

-			log.Printf("error parsing %v", action)

-			return

-		}

-	}

-	var buf []byte

-	var contentType opossum.ContentType

-	if method == "GET" {

-		q := uri.Query()

-		for k, vs := range formData(*form) {

-			log.Printf("add query info %v => %v", k, vs[0])

-			q.Set(k, vs[0]) // TODO: what is with the rest?

-		}

-		uri.RawQuery = q.Encode()

-		log.Printf("uri raw query=%v", uri.RawQuery)

-		buf, contentType, err = b.get(uri, true)

-		log.Printf("uri=%v", uri.String())

-	} else {

-		buf, contentType, err = b.PostForm(uri, formData(*form))

-	}

-	if err == nil {

-		if contentType.IsHTML() {

-			b.render(buf)

-		} else {

-			log.Fatalf("post: unexpected %v", contentType)

-		}

-	} else {

-		log.Printf("post form: %v", err)

-	}

-}

-

 func Arrange(cs style.Map, elements ...*Element) *Element {

 	if cs.IsFlex() {

 		if cs.IsFlexDirectionRow() {

@@ -979,10 +920,6 @@

 	}

 }

-type Website struct {

-	duit.UI

-}

-

 func TraverseTree(ui duit.UI, f func(ui duit.UI)) {

 	traverseTree(0, ui, f)

 }

@@ -1113,7 +1050,6 @@

 type Browser struct {

 	History

 	dui       *duit.DUI

-	html      string

 	Website   *Website

 	StatusBar *duit.Label

 	LocationField *duit.Field

@@ -1158,7 +1094,7 @@

 	if err != nil {

 		log.Fatalf("get: %v", err)

 	}

-	b.html = string(buf)

+	b.Website.html = string(buf)

 	browser = b

 	style.SetFetcher(b)

@@ -1165,7 +1101,7 @@

 	dui = _dui

 	display = dui.Display

-	b.layoutWebsite()

+	b.Website.layout(b)

 	return

 }

@@ -1279,8 +1215,8 @@

 	})

 	imageCache = make(map[string]*draw.Image)

-	b.html = string(buf) // TODO: correctly interpret UTF8

-	b.layoutWebsite()

+	b.Website.html = string(buf) // TODO: correctly interpret UTF8

+	b.Website.layout(b)

 	dui.MarkLayout(dui.Top.UI)

 	dui.MarkDraw(dui.Top.UI)

@@ -1381,137 +1317,3 @@

 	return

 }

-func (b *Browser) layoutWebsite() {

-	pass := func(htm string, csss ...string) (*html.Node, map[*html.Node]style.Map) {

-

-		if debugPrintHtml {

-			log.Printf("%v\n", htm)

-		}

-

-		var doc *html.Node

-		var err error

-		doc, err = html.ParseWithOptions(

-			strings.NewReader(htm),

-			html.ParseOptionEnableScripting(*ExperimentalJsInsecure),

-		)

-		if err != nil {

-			panic(err.Error())

-		}

-

-		log.Printf("Retrieving CSS Rules...")

-		var cssSize int

-		nodeMap := make(map[*html.Node]style.Map)

-		for i, css := range csss {

-

-			log.Printf("CSS size %v kB", cssSize/1024)

-

-			nm, err := style.FetchNodeMap(doc, css, 1280)

-			if err == nil {

-				log.Printf("[%v/%v] Fetch CSS Rules successful!", i+1, len(csss))

-				if debugPrintHtml {

-					log.Printf("%v", nm)

-				}

-				style.MergeNodeMaps(nodeMap, nm)

-			} else {

-				log.Errorf("Fetch CSS Rules failed: %v", err)

-				if *DebugDumpCSS {

-					ioutil.WriteFile("info.css", []byte(css), 0644)

-				}

-			}

-		}

-

-		return doc, nodeMap

-	}

-

-	log.Printf("1st pass")

-	doc, _ := pass(b.html)

-

-	log.Printf("2nd pass")

-	log.Printf("Download style...")

-	cssHrefs := style.Hrefs(doc)

-	csss := make([]string, 0, len(cssHrefs))

-	for _, href := range cssHrefs {

-		url, err := b.LinkedUrl(href)

-		if err != nil {

-			log.Printf("error parsing %v", href)

-			continue

-		}

-		log.Printf("Download %v", url)

-		buf, contentType, err := b.Get(url)

-		if err != nil {

-			log.Printf("error downloading %v", url)

-			continue

-		}

-		if contentType.IsCSS() {

-			csss = append(csss, string(buf))

-		} else {

-			log.Printf("css: unexpected %v", contentType)

-		}

-	}

-	csss = append([]string{ /*string(revertCSS), */ style.AddOnCSS}, csss...)

-	doc, nodeMap := pass(b.html, csss...)

-

-	if *ExperimentalJsInsecure {

-		log.Printf("3rd pass")

-		nt := nodes.NewNodeTree(doc, style.Map{}, nodeMap, nil)

-		jsSrcs := domino.Srcs(nt)

-		downloads := make(map[string]string)

-		for _, src := range jsSrcs {

-			url, err := b.LinkedUrl(src)

-			if err != nil {

-				log.Printf("error parsing %v", src)

-				continue

-			}

-			log.Printf("Download %v", url)

-			buf, _, err := b.Get(url)

-			if err != nil {

-				log.Printf("error downloading %v", url)

-				continue

-			}

-			downloads[src] = string(buf)

-		}

-		codes := domino.Scripts(nt, downloads)

-		log.Infof("JS pipeline start")

-		jsProcessed, err := processJS2(b.html, nt, codes)

-		if err == nil {

-			if b.html != jsProcessed {

-				log.Infof("html changed")

-			}

-			b.html = jsProcessed

-			if debugPrintHtml {

-				log.Printf("%v\n", jsProcessed)

-			}

-			doc, nodeMap = pass(b.html, csss...)

-		} else {

-			log.Errorf("JS error: %v", err)

-		}

-		log.Infof("JS pipeline end")

-	}

-	var countHtmlNodes func(*html.Node) int

-	countHtmlNodes = func(n *html.Node) (num int) {

-		num++

-		for c := n.FirstChild; c != nil; c = c.NextSibling {

-			num += countHtmlNodes(c)

-		}

-		return

-	}

-	log.Printf("%v html nodes found...", countHtmlNodes(doc))

-

-	body := grepBody(doc)

-

-	log.Printf("Layout website...")

-	numElements = 0

-	scroller = duit.NewScroll(

-		NodeToBox(0, b, nodes.NewNodeTree(body, style.Map{}, nodeMap, nil)),

-	)

-	b.Website.UI = scroller

-	log.Printf("Layouting done (%v elements created)", numElements)

-	if numElements < 10 {

-		log.Errorf("Less than 10 elements layouted, seems css processing failed. Will layout without css")

-		scroller = duit.NewScroll(

-			NodeToBox(0, b, nodes.NewNodeTree(body, style.Map{}, make(map[*html.Node]style.Map), nil)),

-		)

-		b.Website.UI = scroller

-	}

-	log.Flush()

-}

--- /dev/null

+++ b/browser/website.go

@@ -1,0 +1,209 @@

+package browser

+

+import (

+	"github.com/mjl-/duit"

+	"golang.org/x/net/html"

+	"io/ioutil"

+	"net/url"

+	"opossum"

+	"opossum/domino"

+	"opossum/nodes"

+	"opossum/style"

+	"strings"

+)

+

+type Website struct {

+	duit.UI

+	html      string

+}

+

+func (w *Website) layout(f opossum.Fetcher) {

+	pass := func(htm string, csss ...string) (*html.Node, map[*html.Node]style.Map) {

+

+		if debugPrintHtml {

+			log.Printf("%v\n", htm)

+		}

+

+		var doc *html.Node

+		var err error

+		doc, err = html.ParseWithOptions(

+			strings.NewReader(htm),

+			html.ParseOptionEnableScripting(*ExperimentalJsInsecure),

+		)

+		if err != nil {

+			panic(err.Error())

+		}

+

+		log.Printf("Retrieving CSS Rules...")

+		var cssSize int

+		nodeMap := make(map[*html.Node]style.Map)

+		for i, css := range csss {

+

+			log.Printf("CSS size %v kB", cssSize/1024)

+

+			nm, err := style.FetchNodeMap(doc, css, 1280)

+			if err == nil {

+				log.Printf("[%v/%v] Fetch CSS Rules successful!", i+1, len(csss))

+				if debugPrintHtml {

+					log.Printf("%v", nm)

+				}

+				style.MergeNodeMaps(nodeMap, nm)

+			} else {

+				log.Errorf("Fetch CSS Rules failed: %v", err)

+				if *DebugDumpCSS {

+					ioutil.WriteFile("info.css", []byte(css), 0644)

+				}

+			}

+		}

+

+		return doc, nodeMap

+	}

+

+	log.Printf("1st pass")

+	doc, _ := pass(w.html)

+

+	log.Printf("2nd pass")

+	log.Printf("Download style...")

+	cssHrefs := style.Hrefs(doc)

+	csss := make([]string, 0, len(cssHrefs))

+	for _, href := range cssHrefs {

+		url, err := f.LinkedUrl(href)

+		if err != nil {

+			log.Printf("error parsing %v", href)

+			continue

+		}

+		log.Printf("Download %v", url)

+		buf, contentType, err := f.Get(url)

+		if err != nil {

+			log.Printf("error downloading %v", url)

+			continue

+		}

+		if contentType.IsCSS() {

+			csss = append(csss, string(buf))

+		} else {

+			log.Printf("css: unexpected %v", contentType)

+		}

+	}

+	csss = append([]string{ /*string(revertCSS), */ style.AddOnCSS}, csss...)

+	doc, nodeMap := pass(w.html, csss...)

+

+	if *ExperimentalJsInsecure {

+		log.Printf("3rd pass")

+		nt := nodes.NewNodeTree(doc, style.Map{}, nodeMap, nil)

+		jsSrcs := domino.Srcs(nt)

+		downloads := make(map[string]string)

+		for _, src := range jsSrcs {

+			url, err := f.LinkedUrl(src)

+			if err != nil {

+				log.Printf("error parsing %v", src)

+				continue

+			}

+			log.Printf("Download %v", url)

+			buf, _, err := f.Get(url)

+			if err != nil {

+				log.Printf("error downloading %v", url)

+				continue

+			}

+			downloads[src] = string(buf)

+		}

+		codes := domino.Scripts(nt, downloads)

+		log.Infof("JS pipeline start")

+		jsProcessed, err := processJS2(w.html, nt, codes)

+		if err == nil {

+			if w.html != jsProcessed {

+				log.Infof("html changed")

+			}

+			w.html = jsProcessed

+			if debugPrintHtml {

+				log.Printf("%v\n", jsProcessed)

+			}

+			doc, nodeMap = pass(w.html, csss...)

+		} else {

+			log.Errorf("JS error: %v", err)

+		}

+		log.Infof("JS pipeline end")

+	}

+	var countHtmlNodes func(*html.Node) int

+	countHtmlNodes = func(n *html.Node) (num int) {

+		num++

+		for c := n.FirstChild; c != nil; c = c.NextSibling {

+			num += countHtmlNodes(c)

+		}

+		return

+	}

+	log.Printf("%v html nodes found...", countHtmlNodes(doc))

+

+	body := grepBody(doc)

+

+	log.Printf("Layout website...")

+	numElements = 0

+	scroller = duit.NewScroll(

+		NodeToBox(0, browser, nodes.NewNodeTree(body, style.Map{}, nodeMap, nil)),

+	)

+	w.UI = scroller

+	log.Printf("Layouting done (%v elements created)", numElements)

+	if numElements < 10 {

+		log.Errorf("Less than 10 elements layouted, seems css processing failed. Will layout without css")

+		scroller = duit.NewScroll(

+			NodeToBox(0, browser, nodes.NewNodeTree(body, style.Map{}, make(map[*html.Node]style.Map), nil)),

+		)

+		w.UI = scroller

+	}

+	log.Flush()

+}

+

+func formData(n html.Node) (data url.Values) {

+	data = make(url.Values)

+	if n.Data == "input" {

+		if k := attr(n, "name"); k != "" {

+			data.Set(k, attr(n, "value"))

+		}

+	}

+	for c := n.FirstChild; c != nil; c = c.NextSibling {

+		for k, vs := range formData(*c) {

+			data.Set(k, vs[0]) // TODO: what aboot the rest?

+		}

+	}

+	return

+}

+

+func (b *Browser) submit(form *html.Node) {

+	var err error

+	method := "GET" // TODO

+	if m := attr(*form, "method"); m != "" {

+		method = strings.ToUpper(m)

+	}

+	uri := b.URL()

+	log.Printf("form = %+v", form)

+	if action := attr(*form, "action"); action != "" {

+		uri, err = b.LinkedUrl(action)

+		if err != nil {

+			log.Printf("error parsing %v", action)

+			return

+		}

+	}

+	var buf []byte

+	var contentType opossum.ContentType

+	if method == "GET" {

+		q := uri.Query()

+		for k, vs := range formData(*form) {

+			log.Printf("add query info %v => %v", k, vs[0])

+			q.Set(k, vs[0]) // TODO: what is with the rest?

+		}

+		uri.RawQuery = q.Encode()

+		log.Printf("uri raw query=%v", uri.RawQuery)

+		buf, contentType, err = b.get(uri, true)

+		log.Printf("uri=%v", uri.String())

+	} else {

+		buf, contentType, err = b.PostForm(uri, formData(*form))

+	}

+	if err == nil {

+		if contentType.IsHTML() {

+			b.render(buf)

+		} else {

+			log.Fatalf("post: unexpected %v", contentType)

+		}

+	} else {

+		log.Printf("post form: %v", err)

+	}

+}

--- a/opossum.go

+++ b/opossum.go

@@ -83,7 +83,7 @@

 	}

 	if strings.ToLower(charset) == "iso-8859-1" {

 		r := bytes.NewReader(buf)

-	    cr := charmap.ISO8859_1.NewDecoder().Reader(r)

+		cr := charmap.ISO8859_1.NewDecoder().Reader(r)

 		updated, err := ioutil.ReadAll(cr)

 		if err == nil {