package main

// scanning an HTTP response for phrases

import (
	"http"
	"mahonia.googlecode.com/hg"
	"strings"
)

// phrasesInResponse scans the content of an http.Response for phrases,
// and returns a map of phrases and counts.
func phrasesInResponse(res *http.Response) map[string]int {
	defer res.Body.Close()

	contentType := res.Header.Get("Content-Type")

	wr := newWordReader(res.Body, decoderForContentType(contentType))
	ps := newPhraseScanner()
	ps.scanByte(' ')
	buf := make([]byte, 4096)
	for {
		n, err := wr.Read(buf)
		if err != nil {
			break
		}
		for _, c := range buf[:n] {
			ps.scanByte(c)
		}
	}
	ps.scanByte(' ')

	return ps.tally
}

func decoderForContentType(t string) mahonia.Decoder {
	t = strings.ToLower(t)
	var result mahonia.Decoder

	i := strings.Index(t, "charset=")
	if i != -1 {
		charset := t[i+len("charset="):]
		i = strings.Index(charset, ";")
		if i != -1 {
			charset = charset[:i]
		}
		result = mahonia.NewDecoder(charset)
	}

	if result == nil {
		result = mahonia.FallbackDecoder(mahonia.NewDecoder("UTF-8"), mahonia.NewDecoder("windows-1252"))
	}

	if strings.Contains(t, "html") {
		result = mahonia.FallbackDecoder(mahonia.EntityDecoder(), result)
	}

	return result
}

package main

// scanning an HTTP response for phrases

import (
	"http"
	"mahonia.googlecode.com/hg"
)

// phrasesInResponse scans the content of an http.Response for phrases,
// and returns a map of phrases and counts.
func phrasesInResponse(res *http.Response) map[string]int {
	defer res.Body.Close()
	wr := newWordReader(res.Body, mahonia.NewDecoder("UTF-8")) // TODO: support other encodings, HTML entities
	ps := newPhraseScanner()
	ps.scanByte(' ')
	buf := make([]byte, 4096)
	for {
		n, err := wr.Read(buf)
		if err != nil {
			break
		}
		for _, c := range buf[:n] {
			ps.scanByte(c)
		}
	}
	ps.scanByte(' ')

	return ps.tally
}

--- 
+++ 
@@ -5,13 +5,17 @@
 import (
 	"http"
 	"mahonia.googlecode.com/hg"
+	"strings"
 )
 
 // phrasesInResponse scans the content of an http.Response for phrases,
 // and returns a map of phrases and counts.
 func phrasesInResponse(res *http.Response) map[string]int {
 	defer res.Body.Close()
-	wr := newWordReader(res.Body, mahonia.NewDecoder("UTF-8")) // TODO: support other encodings, HTML entities
+
+	contentType := res.Header.Get("Content-Type")
+
+	wr := newWordReader(res.Body, decoderForContentType(contentType))
 	ps := newPhraseScanner()
 	ps.scanByte(' ')
 	buf := make([]byte, 4096)
@@ -28,3 +32,28 @@
 
 	return ps.tally
 }
+
+func decoderForContentType(t string) mahonia.Decoder {
+	t = strings.ToLower(t)
+	var result mahonia.Decoder
+
+	i := strings.Index(t, "charset=")
+	if i != -1 {
+		charset := t[i+len("charset="):]
+		i = strings.Index(charset, ";")
+		if i != -1 {
+			charset = charset[:i]
+		}
+		result = mahonia.NewDecoder(charset)
+	}
+
+	if result == nil {
+		result = mahonia.FallbackDecoder(mahonia.NewDecoder("UTF-8"), mahonia.NewDecoder("windows-1252"))
+	}
+
+	if strings.Contains(t, "html") {
+		result = mahonia.FallbackDecoder(mahonia.EntityDecoder(), result)
+	}
+
+	return result
+}
Use charset from Content-Type header instead of assuming all pages are UTF-8.
