Program

# -*- coding: utf-8 -*-

import urllib, httplib, urlparse
import re
import random

"""judge url exists or not,by others"""
def httpExists(url):
    host, path = urlparse.urlsplit(url)[1:3]
    if ':' in host:
        # port specified, try to use it
        host, port = host.split(':', 1)
        try:
            port = int(port)
        except ValueError:
            print 'invalid port number %r' % (port,)
            return False
    else:
        # no port specified, use default port
        port = None
    try:
        connection = httplib.HTTPConnection(host, port=port)
        connection.request("HEAD", path)
        resp = connection.getresponse( )
        if resp.status == 200:       # normal 'found' status
            found = True
        elif resp.status == 302:     # recurse on temporary redirect
            found = httpExists(urlparse.urljoin(url,resp.getheader('location', '')))
        else:                        # everything else -> not found
            print "Status %d %s : %s" % (resp.status, resp.reason, url)
            found = False
    except Exception, e:
        print e.__class__, e, url
        found = False
    return found

"""get html src,return lines[]"""
def gGetHtmlLines(url):
    if url==None : return
    if not httpExists(url): return 
    try:
        page = urllib.urlopen(url)   
        html = page.readlines()
        page.close()
        return html
    except:
        print "gGetHtmlLines() error!"
        return
"""get html src,return string"""
def gGetHtml(url):
    if url==None : return
    if not httpExists(url): return 
    try:
        page = urllib.urlopen(url)   
        html = page.read()
        page.close()
        return html
    except:
        print "gGetHtml() error!"
        return

"""根据url获取文件名"""
def gGetFileName(url):
    if url==None: return None
    if url=="" : return ""
    arr=url.split("/")
    return arr[len(arr)-1]

"""生成随机文件名"""
def gRandFilename(type):
    fname = ''
    for i in range(16):
        fname = fname + chr(random.randint(65,90))
        fname = fname + chr(random.randint(48,57))
    return fname + '.' + type
"""根据url和其上的link，得到link的绝对地址"""
def gGetAbslLink(url,link):
    if url==None or link == None : return 
    if url=='' or link=='' : return url 
    addr = '' 
    if link[0] == '/' : 
        addr = gGetHttpAddr(url) + link 
    elif len(link)>3 and link[0:4] == 'http':
        addr =  link 
    elif len(link)>2 and link[0:2] == '..':
        addr = gGetHttpAddrFatherAssign(url,link)
    else:
        addr = gGetHttpAddrFather(url) + link 

    return addr 

"""根据输入的lines，匹配正则表达式，返回list"""
def gGetRegList(linesList,regx):
    if linesList==None : return 
    rtnList=[]
    for line in linesList:
        matchs = re.search(regx, line, re.IGNORECASE)
        if matchs!=None:
            allGroups = matchs.groups()
            for foundStr in allGroups:
                if foundStr not in rtnList:
                    rtnList.append(foundStr)
    return rtnList
"""根据url下载文件，文件名参数指定"""
def gDownloadWithFilename(url,savePath,file):
    #参数检查，现忽略
    try:
        urlopen=urllib.URLopener()
        fp = urlopen.open(url)
        data = fp.read()
        fp.close()
        file=open(savePath + file,'w+b')
        file.write(data)
        file.close()
    except IOError:
        print "download error!"+ url
        
"""根据url下载文件，文件名自动从url获取"""
def gDownload(url,savePath):
    #参数检查，现忽略
    fileName = gGetFileName(url)
    #fileName =gRandFilename('jpg')
    gDownloadWithFilename(url,savePath,fileName)
        
"""根据某网页的url,下载该网页的jpg"""
def gDownloadHtmlJpg(downloadUrl,savePath):
    lines= gGetHtmlLines(downloadUrl)
    regx = r"""src\s*="?(\S+)\.jpg"""
    lists =gGetRegList(lines,regx)
    if lists==None: return 
    for jpg in lists:
        jpg = gGetAbslLink(downloadUrl,jpg) + '.jpg'
        gDownload(jpg,savePath)
   ###     print gGetFileName(jpg)
"""根据url取主站地址"""
def gGetHttpAddr(url):
    if url== '' : return ''
    arr=url.split("/")
    return arr[0]+"//"+arr[2]
"""根据url取上级目录"""
def gGetHttpAddrFather(url):
    if url=='' : return ''
    arr=url.split("/")
    addr = arr[0]+'//'+arr[2]+ '/'
    if len(arr)-1>3 :
        for i in range(3,len(arr)-1):
            addr = addr + arr[i] + '/'
    return addr

"""根据url和上级的link取link的绝对地址"""
def gGetHttpAddrFatherAssign(url,link):
    if url=='' : return ''
    if link=='': return ''
    linkArray=link.split("/")
    urlArray = url.split("/")
    partLink =''
    partUrl = ''
    for i in range(len(linkArray)):        
        if linkArray[i]=='..': 
            numOfFather = i + 1    #上级数
        else:
            partLink = partLink + '/'  + linkArray[i]
    for i in range(len(urlArray)-1-numOfFather):
        partUrl = partUrl + urlArray[i] 
        if i < len(urlArray)-1-numOfFather -1 : 
            partUrl = partUrl + '/'
    return  partUrl + partLink

"""根据url获取其上的相关htm、html链接，返回list"""
def gGetHtmlLink(url):
    #参数检查，现忽略
    rtnList=[]
    lines=gGetHtmlLines(url)
    regx = r"""href="?(\S+)\.htm"""
    for link in gGetRegList(lines,regx):
        link = gGetAbslLink(url,link) + '.htm'
        if link not in rtnList:
            rtnList.append(link)
            print link
    return rtnList

"""根据url，抓取其上的jpg和其链接htm上的jpg"""
def gDownloadAllJpg(url,savePath):
    #参数检查，现忽略
    gDownloadHtmlJpg(url,savePath)
    #抓取link上的jpg
    links=gGetHtmlLink(url)
    for link in links:
        gDownloadHtmlJpg(link,savePath)

"""test"""
def test():
    u='http://www.cots.com.cn/index.htm'
    save='d:/tmp/'
    print 'download pic from [' + u +']'
    print 'save to [' +save+'] ...'
    gDownloadHtmlJpg(u,save)
    print "download finished"
    
test()

I'm thinking of trying Beautiful Soup, a Python package for HTML scraping. Are there any other HTML scraping packages I should be looking at? Python is not a requirement, I'm actually interested in hearing about other languages as well.

The story so far:

• Python

   

  • Beautiful Soup
  • lxml
  • HTQL
  • Scrapy
  • Mechanize

• Ruby

   

  • Nokogiri
  • Hpricot
  • Mechanize
  • scrAPI
  • scRUBYt!
  • wombat
  • Watir

• .NET

   

  • Html Agility Pack
  • WatiN

• Perl

   

  • WWW::Mechanize
  • Web-Scraper

• Java

   

  • Tag Soup
  • HtmlUnit
  • Web-Harvest
  • jARVEST
  • jsoup
  • Jericho HTML Parser

• JavaScript

   

  • node.io
  • phantomjs

• PHP

   

  • htmlSQL

• Most of them

   

  • Screen-Scraper

• Web Service

   

  • Bobik

1:安装 http://www.pythonware.com/products/pil/
2:安装 https://code.google.com/p/pytesser/downloads/detail?name=pytesser_v0.0.1.zip&can=2&q=

测试结果 pytesser 识别率太低

1:取得50个样例图
import httplib
for i in range(50):
    url = 'http:////sysmonitor/verifyCodeServlet'
    print "download", i
    c = httplib.HTTPSConnection("1.1.1.1",8443)
    c.request("GET", "/sysmonitor/verifyCodeServlet")
    response = c.getresponse()
    #print response.status, response.reason
    #data = response.read()
    file("./code/%04d.png" % i, "wb").write(response.read())

2:去噪
from PIL import Image

for i in range(50):
 
    img = Image.open("./code/%04d.png" % i)
    img = img.convert("RGBA")
 
    pixdata = img.load()
 

    for y in xrange(img.size[1]):
        for x in xrange(img.size[0]):
            if pixdata[x, y][0] < 90:
                pixdata[x, y] = (0, 0, 0, 255)
 
    for y in xrange(img.size[1]):
        for x in xrange(img.size[0]):
            if pixdata[x, y][1] < 136:
                pixdata[x, y] = (0, 0, 0, 255)
 
    for y in xrange(img.size[1]):
        for x in xrange(img.size[0]):
            if pixdata[x, y][2] > 0:
                pixdata[x, y] = (255, 255, 255, 255)
 
    img.save("./code/input-black-%04d.gif" % i, "GIF")
 

#im_orig = Image.open('input-black.gif')
#big = im_orig.resize((1000, 500), Image.NEAREST)

3:生成字库
import os ,Image
j = 1
dir="./code/"
for f in os.listdir(dir):
    if f.endswith(".gif"):
        img = Image.open(dir+f)
        for i in range(4): 
            x = 4 + i*18
            y = 2
            img.crop((x, y, x+12, y+16)).save("fonts/%d.gif" % j)
            print "j=",j 
            j += 1 

4:识别
#!/usr/bin/env python
# ?*? coding: UTF?8 ?*?
import os, Image
 
def binary(f):
    img = Image.open(f)
    #img = img.convert('1')
    img = img.convert("RGBA")
    pixdata = img.load()
    for y in xrange(img.size[1]):
        for x in xrange(img.size[0]):
            if pixdata[x, y][0] < 90:
                pixdata[x, y] = (0, 0, 0, 255)
    for y in xrange(img.size[1]):
        for x in xrange(img.size[0]):
            if pixdata[x, y][1] < 136:
                pixdata[x, y] = (0, 0, 0, 255)
    for y in xrange(img.size[1]):
        for x in xrange(img.size[0]):
            if pixdata[x, y][2] > 0:
                pixdata[x, y] = (255, 255, 255, 255)
    return img
 
 
def division(img):
    font=[]
    for i in range(4):
        x=4 + i*18
        y=2
        font.append(img.crop((x,y,x+12,y+16)))
    return font
 
def recognize(img):
    fontMods = []
    for i in range(10):
        fontMods.append((str(i), Image.open("./num/%d.gif" % i)))
    result=""
    font=division(img)
    for i in font:
        target=i
        points = []
        for mod in fontMods:
            diffs = 0
            for yi in range(16):
                for xi in range(12):
                    #(mod[1].getpixel((xi, yi))).save("./temp/temp.gif" % j)
                    if mod[1].getpixel((xi, yi)) != target.getpixel((xi, yi)):
                        #diffs += 1
                        if 0 in target.getpixel((xi, yi)):
                            compare = 0
                        else:
                            compare = 255

                        if mod[1].getpixel((xi, yi)) != compare:
                            diffs += 1
                            
            print "diffs：" + str(diffs)
            points.append((diffs, mod[0]))
        points.sort()
        result += points[0][1]
    return result
 
if __name__ == '__main__':
    codedir="./code/"
    for imgfile in os.listdir(codedir):
        if imgfile.endswith(".png"):
            dir="./result/"
            print(codedir+imgfile)
            img=binary(codedir+imgfile)
            num=recognize(img)
            dir += (num+".png")
            print "save to", dir
            img.save(dir)

UnicodeDecodeError: 'ascii' codec can't decode byte 0xb0 in position 1: ordinal not in range(128) 原因可能与注册表错误的字符集有关，可能与某些软件对注册表的改写的gbk格式导致。

解决方法：打开C:\Python27\Lib下的 mimetypes.py 文件，

找到大概256行的 ‘default_encoding = sys.getdefaultencoding()’。

在这行前面添加三行：

if sys.getdefaultencoding() != 'gbk':  
    reload(sys)  
    sys.setdefaultencoding('gbk')  
default_encoding = sys.getdefaultencoding()  

首先安装 PyCrypto – The Python Cryptography Toolkit
http://www.voidspace.org.uk/python/modules.shtml#pycrypto

https://github.com/paramiko/paramiko
setup.py install

如出现 error: Unable to find vcvarsall.bat
则:
命令行下执行 SET VS90COMNTOOLS=%VS100COMNTOOLS%
如果你安装的是 2012 版 SET VS90COMNTOOLS=%VS110COMNTOOLS%
如果你安装的是 2013版 SET VS90COMNTOOLS=%VS120COMNTOOLS%

1.java

[code lang=”java”]<br />
package com.egame.fee.sdk.pc.util;
</p>

<p>
import java.io.IOException;<br />
import java.io.UnsupportedEncodingException;<br />
import java.security.InvalidKeyException;<br />
import java.security.NoSuchAlgorithmException;<br />
import java.security.SecureRandom;<br />
import java.security.spec.InvalidKeySpecException;
</p>

<p>
import javax.crypto.BadPaddingException;<br />
import javax.crypto.Cipher;<br />
import javax.crypto.IllegalBlockSizeException;<br />
import javax.crypto.NoSuchPaddingException;<br />
import javax.crypto.SecretKey;<br />
import javax.crypto.SecretKeyFactory;<br />
import javax.crypto.spec.DESKeySpec;
</p>

<p>
import sun.misc.BASE64Decoder;<br />
import sun.misc.BASE64Encoder;
</p>

<p>
public class SecretUtilTools {
</p>

<p>
public static String encryptForDES(String souce, String key) throws InvalidKeyException,<br />
NoSuchAlgorithmException, InvalidKeySpecException, NoSuchPaddingException,<br />
IllegalBlockSizeException, BadPaddingException, UnsupportedEncodingException {<br />
// DES算法要求有一个可信任的随机数源<br />
SecureRandom sr = new SecureRandom();<br />
// 从原始密匙数据创建DESKeySpec对象<br />
DESKeySpec dks = new DESKeySpec(key.getBytes(&quot;UTF-8&quot;));<br />
// 创建一个密匙工厂，然后用它把DESKeySpec转换成 一个SecretKey对象<br />
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(&quot;DES&quot;);<br />
SecretKey key1 = keyFactory.generateSecret(dks);<br />
// Cipher对象实际完成加密操作<br />
Cipher cipher = Cipher.getInstance(&quot;DES&quot;);<br />
// 用密匙初始化Cipher对象<br />
cipher.init(Cipher.ENCRYPT_MODE, key1, sr);<br />
// 现在，获取数据并加密<br />
byte encryptedData[] = cipher.doFinal(souce.getBytes(&quot;UTF-8&quot;));<br />
// 通过BASE64位编码成字符创形式<br />
String base64Str = new BASE64Encoder().encode(encryptedData);
</p>

<p>
return base64Str;<br />
}
</p>

<p>
public static String decryptForDES(String souce, String key) throws InvalidKeyException,<br />
NoSuchAlgorithmException, InvalidKeySpecException, NoSuchPaddingException, IOException,<br />
IllegalBlockSizeException, BadPaddingException {<br />
// DES算法要求有一个可信任的随机数源<br />
SecureRandom sr = new SecureRandom();<br />
// 从原始密匙数据创建DESKeySpec对象<br />
DESKeySpec dks = new DESKeySpec(key.getBytes());<br />
// 创建一个密匙工厂，然后用它把DESKeySpec转换成 一个SecretKey对象<br />
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(&quot;DES&quot;);<br />
SecretKey key1 = keyFactory.generateSecret(dks);<br />
// Cipher对象实际完成加密操作<br />
Cipher cipher = Cipher.getInstance(&quot;DES&quot;);<br />
// 用密匙初始化Cipher对象<br />
cipher.init(Cipher.DECRYPT_MODE, key1, sr);<br />
// 将加密报文用BASE64算法转化为字节数组<br />
byte[] encryptedData = new BASE64Decoder().decodeBuffer(souce);<br />
// 用DES算法解密报文<br />
byte decryptedData[] = cipher.doFinal(encryptedData);<br />
return new String(decryptedData,&quot;UTF-8&quot;);<br />
}
</p>

<p>
}<br />
[/code]

2.php

[code lang=”php”]<br />
&lt;？php
</p>

<p>
class SecretUtilTools<br />
{<br />
//加密算法<br />
function encryptForDES(input,key)<br />
{<br />
size = mcrypt_get_block_size(&#39;des&#39;,&#39;ecb&#39;);<br />input = this-&amp;gt;pkcs5_pad(input, size);<br />td = mcrypt_module_open(&#39;des&#39;, &#39;&#39;, &#39;ecb&#39;, &#39;&#39;);<br />
iv = @mcrypt_create_iv (mcrypt_enc_get_iv_size(td), MCRYPT_RAND);<br />
@mcrypt_generic_init(td,key, iv);<br />data = mcrypt_generic(td,input);<br />
mcrypt_generic_deinit(td);<br />
mcrypt_module_close(td);<br />
data = base64_encode(data);<br />
return data;<br />
}<br />
//解密算法<br />
function decryptForDES(encrypted,key)<br />
{<br />encrypted = base64_decode(encrypted);<br />td = mcrypt_module_open(&#39;des&#39;,&#39;&#39;,&#39;ecb&#39;,&#39;&#39;);<br />
//使用MCRYPT_DES算法,cbc模式<br />
iv = @mcrypt_create_iv(mcrypt_enc_get_iv_size(td), MCRYPT_RAND);<br />
ks = mcrypt_enc_get_key_size(td);<br />
@mcrypt_generic_init(td,key, iv);<br />
//初始处理<br />decrypted = mdecrypt_generic(td,encrypted);<br />
//解密<br />
mcrypt_generic_deinit(td);<br />
//结束<br />
mcrypt_module_close(td);<br />
y=this-&amp;gt;pkcs5_unpad(decrypted);<br />
returny;<br />
}
</p>

<p>
function pkcs5_pad (text,blocksize)<br />
{<br />
pad =blocksize – (strlen(text) %blocksize);<br />
return text . str_repeat(chr(pad), pad);<br />
}
</p>

<p>
function pkcs5_unpad(text)<br />
{<br />
pad = ord(text{strlen(text)-1});<br />
if (pad &amp;gt; strlen(text))<br />
{<br />
return false;<br />
}<br />
if (strspn(text, chr(pad), strlen(text) – pad) !=pad)<br />
{<br />
return false;<br />
}<br />
return substr(text, 0, -1 *pad);<br />
}<br />
}<br />
？&gt;[/code]

3.c#

[code lang=”csharp”]<br />
using System;<br />
using System.IO;<br />
using System.Security.Cryptography;<br />
using System.Text;<br />
using System.Threading;
</p>

<p>
public class SecretUtilTools<br />
{<br />
public string encryptForDES(string message, string key)<br />
{<br />
using (DESCryptoServiceProvider des = new DESCryptoServiceProvider())<br />
{<br />
byte[] inputByteArray = Encoding.UTF8.GetBytes(message);<br />
des.Key = UTF8Encoding.UTF8.GetBytes(key);<br />
des.IV = UTF8Encoding.UTF8.GetBytes(key);<br />
des.Mode = System.Security.Cryptography.CipherMode.ECB;<br />
System.IO.MemoryStream ms = new System.IO.MemoryStream();<br />
using (CryptoStream cs = new CryptoStream(ms, des.CreateEncryptor(), CryptoStreamMode.Write))<br />
{<br />
cs.Write(inputByteArray, 0, inputByteArray.Length);<br />
cs.FlushFinalBlock();<br />
cs.Close();<br />
}<br />
string str = Convert.ToBase64String(ms.ToArray());<br />
ms.Close();<br />
return str;<br />
}<br />
}
</p>

<p>
public string decryptForDES(string message, string key)<br />
{<br />
byte[] inputByteArray = Convert.FromBase64String(message);<br />
using (DESCryptoServiceProvider des = new DESCryptoServiceProvider())<br />
{<br />
des.Key = UTF8Encoding.UTF8.GetBytes(key);<br />
des.IV = UTF8Encoding.UTF8.GetBytes(key);<br />
des.Mode = System.Security.Cryptography.CipherMode.ECB;<br />
System.IO.MemoryStream ms = new System.IO.MemoryStream();<br />
using (CryptoStream cs = new CryptoStream(ms, des.CreateDecryptor(), CryptoStreamMode.Write))<br />
{<br />
cs.Write(inputByteArray, 0, inputByteArray.Length);<br />
cs.FlushFinalBlock();<br />
cs.Close();<br />
}<br />
string str = Encoding.UTF8.GetString(ms.ToArray());<br />
ms.Close();<br />
return str;<br />
}<br />
}
</p>

<p>
}<br />
[/code]