目前NBS上有2015-2018四个年度的代码信息，写一个控制台程序爬一下县级行政区下的代码。
使用HttpWebRequest+HttpWebResponse获取html，使用HtmlAgilityPack类库解析HTML。
使用POST请求，请求头带Cookie信息，否则会被反爬机制挡死，返回“请开启JavaScript并刷新该页”。
县级URL Request获取数据的同时记录Response的Cookie信息，在请求镇级数据时，请求头发送此cookie。
“省-地-县-乡 ”与“省-县（地）-乡” 的URL长度不同，根据长度判断URL正确性时需注意，也许还有其他可能，暂未发现。
主方法
class Program
{
static void Main(string[] args)
{
Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine("\r\n----获取县级行政区乡、村二级区划代码");
Console.WriteLine("----数据年份有：");
Console.ResetColor();
Cursor.WriteAt("A、2018", , );
Cursor.WriteAt("B、2017", , );
Cursor.WriteAt("C、2016", , );
Cursor.WriteAt("D、2015", , );
Input: Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine();
Console.WriteLine("----请输入一个年份代码（回车提交）：");
Console.ResetColor();
char chr = Convert.ToChar( Console.ReadLine().ToLower()[]);
if ((int)chr >= &&(int)chr <= )
{
string year = string.Empty;
switch (chr)
{
case 'a':
year = ""; break;
case 'b':
year = ""; break;
case 'c':
year = ""; break;
default:
year = ""; break;
}
System.Diagnostics.Process.Start($"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/{year}");
Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine("浏览器已加载区划代码起始页，请进入县级行政单位页面，复制url，粘贴到下面（回车提交）：");
}
else
goto Input;
Console.ResetColor();
string cityurl = Console.ReadLine();
if (cityurl.Length != && cityurl.Length!=)
{
Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine("url有误，请确认是县级行政单位页面，重新复制链接，粘贴到下面：");
Console.ResetColor();
cityurl = Console.ReadLine();
}
try
{
Console.ForegroundColor = ConsoleColor.Magenta;
Func<object, List<TownInfo>> func = new Func<object, List<TownInfo>>(GetTownInfos);
Task<List<TownInfo>> task = new Task<List<TownInfo>>(func, cityurl);
task.Start();
task.Wait();
if (task.Status == TaskStatus.RanToCompletion && task.Result.Count > )
{ List<VillageInfo> villageInfos = new List<VillageInfo>();
foreach (var item in task.Result)
{
//把乡镇信息写入村级列表，实现乡镇信息输出
VillageInfo villageInfo_town = new VillageInfo(item.Code, "", item.Name);
villageInfos.Add(villageInfo_town);
Func<object, List<VillageInfo>> func1 = new Func<object, List<VillageInfo>>(GetVillageInfos);
Task<List<VillageInfo>> task1 = new Task<List<VillageInfo>>(func1, item.Href);
task1.Start();
task1.Wait();
if (task1.Status == TaskStatus.RanToCompletion)
{
villageInfos.AddRange(task1.Result);
}
}
foreach (var item1 in villageInfos)
{
Console.WriteLine($"{item1.Name.Trim()}\t{item1.Cls.Trim()}\t{item1.Code.Trim()}");
}
}
else
{ Console.WriteLine("乡镇列表获取失败！"); } }
catch (Exception)
{
throw new Exception("");
}
Console.ReadKey();
}
static string cookies = "AD_RS_COOKIE=20082854; wzws_cid=453a2d88181321410de83ba7eedaba3a141eb61ee7488027b6ab07a66054605e99e886827afa72708ce170398ea2fdfeec55455a7c0be8e779694026255f2166";
//获取乡镇级信息列表
static List<TownInfo> GetTownInfos(object cityurl)
{
List<TownInfo> townInfos = new List<TownInfo>();
HttpGetHelper httpGetHelper = new HttpGetHelper() { Url =(string) cityurl, ContentType = "text/html; charset=gb2312", Encode = Encoding.GetEncoding(),RequestMethod="post"};
//HtmlAgilityPack类库解析HTML
HtmlDocument document = new HtmlDocument();
document.LoadHtml(httpGetHelper.GetHtml(,ref cookies));
//string html = httpGetHelper.GetHtml(ref cookies);
//路径里"//"表示从根节点开始查找，两个斜杠‘//’表示查找所有childnodes；一个斜杠'/'表示只查找第一层的childnodes（即不查找grandchild）；点斜杠"./"表示从当前结点而不是根结点开始查找
HtmlNodeCollection htmlNodes = document.DocumentNode.SelectNodes("//tr[@class='towntr']");
foreach (var node in htmlNodes)
{
HtmlNodeCollection htmlNodes1 = node.SelectNodes("./td");
HtmlNode htmlNodeHref = node.SelectSingleNode(".//a[@href]");
HtmlAttribute htmlAttribute = htmlNodeHref.Attributes["href"];
TownInfo townInfo = new TownInfo(htmlNodes1[].InnerText, htmlNodes1[].InnerText,
(cityurl as string).Substring(, (cityurl as string).LastIndexOf('/') + ) + htmlAttribute.Value);
townInfos.Add(townInfo);
}
return townInfos;
}
//获取村级信息列表
static List<VillageInfo> GetVillageInfos(object townurl)
{
List<VillageInfo> villageInfos = new List<VillageInfo>();
HttpGetHelper httpGetHelper = new HttpGetHelper() { Url = (string)townurl, ContentType = "text/html; charset=gb2312", Encode = Encoding.GetEncoding(), RequestMethod = "post"};
HtmlDocument document = new HtmlDocument();
document.LoadHtml(httpGetHelper.GetHtml(,ref cookies));
//string html = httpGetHelper.GetHtml(ref cookies);
HtmlNodeCollection htmlNodes = document.DocumentNode.SelectNodes("//tr[@class='villagetr']");
foreach (var node in htmlNodes)
{
HtmlNodeCollection htmlNodes1 = node.SelectNodes(".//td");
VillageInfo villageInfo = new VillageInfo(htmlNodes1[].InnerText,htmlNodes1[].InnerText,htmlNodes1[].InnerText);
villageInfos.Add(villageInfo);
}
return villageInfos;
}
}
辅助类/结构
internal class Cursor
{
const int origRow = ;
const int origCol = ;
public static void WriteAt(string s, int c, int r)
{
Console.SetCursorPosition(origCol + c, origRow + r);
Console.Write(s);
}
}
//乡镇信息结构 编码、名称、超链
struct TownInfo
{
string code;
public string Code{ get { return code; } }
string name;
public string Name{get { return name; } }
string href;
public string Href { get { return href; } }
public TownInfo (string code,string name,string href)
{
this.code = code;
this.name = name;
this.href = href;
}
}
//村信息结构 编码、城乡划分类，名称
struct VillageInfo
{
string code;
public string Code{ get { return code; } }
string cls;
public string Cls{ get { return cls; } }
string name;
public string Name{ get { return name; } }
public VillageInfo(string code,string cls,string name)
{
this.code = code;
this.cls = cls;
this.name = name;
}
}
获取HTML
public class HttpGetHelper
{
string url = string.Empty;
public string Url
{
set { url = value; }
} int timeOut=*;
public int Timeout
{
set { timeOut = value; }
} string contentType= "text/html;charset=utf-8";
public string ContentType
{
set { contentType = value; }
} string userAgent= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 ";
public string UserAgent
{
set { userAgent = value; }
} Encoding encode=Encoding.UTF8;
public Encoding Encode
{
set { encode = value; }
}
string request_Method = "get";
public string RequestMethod
{
set { request_Method = value; }
}
/// <summary>
/// get html content
/// </summary>
/// <param name="cls">town=1;village=2</param>
/// <param name="cookies">if cls=1 then ref cookies</param>
/// <returns></returns>
public string GetHtml(int cls,ref string cookies)
{
string html = string.Empty;
try
{
if (url!=string.Empty)
{
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
request.Timeout = this.timeOut;
request.ContentType = this.contentType;
request.UserAgent = this.userAgent;
request.Headers.Add(HttpRequestHeader.Cookie, cookies);
request.Method = request_Method;
using (HttpWebResponse response =request.GetResponse()as HttpWebResponse)
{
if (response.StatusCode==HttpStatusCode.OK)
{//如果是县级url，则记录cookie
if (cls==)
{
CookieCollection cookieCollection = response.Cookies;
foreach (Cookie item in cookieCollection)
{
cookies = item.Name + "=" + item.Value + ";";
}
cookies.Remove(cookies.Length - );
} using (StreamReader streamReader = new StreamReader(response.GetResponseStream(), encode))
{
html = streamReader.ReadToEnd();
streamReader.Close();
}
}
}
}
}
catch (Exception)
{
throw new Exception($"GetHtml失败，url:{url}");
}
return html;
}
}