homeworkSearchEngine/function.groovy at master · Mr-win7/homeworkSearchEngine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import java.util.List

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
class Url
{
def public static unicodeToUtf8(theString)
{

}
def isUTF_8(content)
{
	def regex='<meta .*?charset.*?(utf|UTF)-8[^\\<\\>]*?>'
	return (content=~regex).find()
}
def getChilds(content)
{
	def regex='<a .*? href=[\'"](http://[^"\']*\\.sina\\.com\\.cn[^"\']*).*?>'
	def matcher=content=~regex
	return matcher
}
def getContent_2(url)
{

	def content
	def httpclient=HttpClients.createDefault()
	def httpget=new HttpGet(url)
	try {
		def response = httpclient.execute(httpget)
		def entity=response.getEntity()
		if (entity != null)
		{
			def instream = entity.getContent()
			def tmp
			def result=new StringBuffer()
			def reader=new BufferedReader(new InputStreamReader(instream,"ISO-8859-1"))
			while((tmp=reader.readLine())!=null)
			{
				result.append(tmp)
			}
			tmp=result.toString()
			content=new String(tmp.getBytes("ISO-8859-1"),"GB2312")
			instream.close()
		}
		response.close()
	} catch (e) {
		// TODO Auto-generated catch block
		e.printStackTrace()
	}

		return content
}
def getContent_1(url)
{
	def i=-1
	def baos=new ByteArrayOutputStream()
	def httpclient=HttpClients.createDefault()
	def httpget=new HttpGet(url)
	try {
		def response = httpclient.execute(httpget)
		def entity=response.getEntity()
		if (entity != null)
		{
			def instream = entity.getContent()
			while((i=instream.read())!=-1)
			{
				baos.write(i)
			}
			instream.close()
		}
		response.close()
	} catch (e) {
		// TODO Auto-generated catch block
		e.printStackTrace()
	}
	def content=baos.toString()
	if(!isUTF_8(content))
	{
		return getContent_2(url)
	}
	else
	{
		return content
	}
}
def getTitle_1(content)
{
	def regex='\\<(title|TITLE)\\>([\\s\\S]*?)\\</(title|TITLE)\\>'
	def matcher=content=~regex
	return matcher
}


	def Url(path)
	{
		this.path=path
	}
	def title
	def path
	def content
}
	def i=1
	def urlList=["http://www.sina.com.cn"]
	def queue= [new Url("http://www.sina.com.cn")]
	while(!queue.isEmpty())
	{
		println(queue[0].getPath())
		println(i)
		i++
		try{
		queue[0].content=queue[0].getContent_1(queue[0].path)
		queue[0].title=queue[0].getTitle_1(queue[0].content)[0][2].replace("\n","")
		println(queue[0].title)
		}catch(e){
		println("error")
		queue.remove(0)
		continue
		}

		def matcher_1=queue[0].getChilds(queue[0].getContent())
		for(ma in matcher_1)
		{

			def str=ma[1].replace(" ","").replace("\n","")
			if(!urlList.contains(str))
			{
//				println(str)
				queue.add(new Url(str))
				urlList.add(str)
//				println(i)
//				i++
			}
		}
		queue.remove(0)
	}

interface UrlMapper
{
	def insertUrl(url)
	def selectPathAll()
	def selectUrl(search,pattern)
}