C#爬蟲基礎之HttpClient獲取HTTP請求與響應
一、概述
Net4.5以上的提供基本類,用于發(fā)送 HTTP 請求和接收來自通過 URI 確認的資源的 HTTP 響應。
HttpClient是一個高級 API,用于包裝其運行的每個平臺上可用的較低級別功能。
// HttpClient is intended to be instantiated once per application, rather than per-use. See Remarks. static readonly HttpClient client = new HttpClient(); static async Task Main() { // Call asynchronous network methods in a try/catch block to handle exceptions. try { HttpResponseMessage response = await client.GetAsync("http://www.contoso.com/"); response.EnsureSuccessStatusCode(); string responseBody = await response.Content.ReadAsStringAsync(); // Above three lines can be replaced with new helper method below // string responseBody = await client.GetStringAsync(uri); Console.WriteLine(responseBody); } catch(HttpRequestException e) { Console.WriteLine("\nException Caught!"); Console.WriteLine("Message :{0} ",e.Message); } }
二、HttpClient的使用
1.使用HttpClient調用Oauth的授權接口獲取access_token
1)OAuth使用的密碼式
2)獲取到access_token后才進行下一步
2.帶著access_token調用接口
1)hearder上添加bearer方式的access_token
2)調用接口確保成功獲取到返回的結果
try { string host = ConfigurationManager.AppSettings["api_host"]; string username = ConfigurationManager.AppSettings["api_username"]; string password = ConfigurationManager.AppSettings["api_password"]; HttpClient httpClient = new HttpClient(); // 設置請求頭信息 httpClient.DefaultRequestHeaders.Add("Host", host); httpClient.DefaultRequestHeaders.Add("Method", "Post"); httpClient.DefaultRequestHeaders.Add("KeepAlive", "false"); // HTTP KeepAlive設為false,防止HTTP連接保持 httpClient.DefaultRequestHeaders.Add("UserAgent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); //獲取token var tokenResponse = httpClient.PostAsync("http://" + host + "/token", new FormUrlEncodedContent(new Dictionary<string, string> { {"grant_type","password"}, {"username", username}, {"password", password} })); tokenResponse.Wait(); tokenResponse.Result.EnsureSuccessStatusCode(); var tokenRes = tokenResponse.Result.Content.ReadAsStringAsync(); tokenRes.Wait(); var token = Newtonsoft.Json.Linq.JObject.Parse(tokenRes.Result); var access_token = token["access_token"].ToString(); // 調用接口發(fā)起POST請求 var authenticationHeaderValue = new AuthenticationHeaderValue("bearer", access_token); httpClient.DefaultRequestHeaders.Authorization = authenticationHeaderValue; var content = new StringContent(parameter); content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); var response = httpClient.PostAsync("http://" + host + "/" + api_address, content); response.Wait(); response.Result.EnsureSuccessStatusCode(); var res = response.Result.Content.ReadAsStringAsync(); res.Wait();return Newtonsoft.Json.JsonConvert.DeserializeObject(res.Result); } catch (Exception ex) { return ResultEx.Init(ex.Message); }
HttpClient 獲取圖片并保存到本機
class Program { static void Main() { //圖片路徑:https://img.infinitynewtab.com/wallpaper/1.jpg string imgSourceURL = "https://img.infinitynewtab.com/wallpaper/"; DownloadImags(imgSourceURL).Wait(); } private static async Task DownloadImags(string url) { var client = new HttpClient(); System.IO.FileStream fs; int a = 1; //文件名:序號+.jpg??芍付ǚ秶?,以下是獲取100.jpg~500.jpg. for (int i = 100; i <= 500; i++) { var uri = new Uri(Uri.EscapeUriString(url+i.ToString()+".jpg")); byte[] urlContents = await client.GetByteArrayAsync(uri); fs = new System.IO.FileStream(AppDomain.CurrentDomain.BaseDirectory + "\\images\\"+ i.ToString() + ".jpg",System.IO.FileMode.CreateNew); fs.Write(urlContents, 0, urlContents.Length); Console.WriteLine(a++); } } }
以下為封裝的類庫
using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; using System.Xml.Serialization; public class HttpClientHelpClass { /// /// get請求 /// /// /// public static string GetResponse(string url, out string statusCode) { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json")); HttpResponseMessage response = httpClient.GetAsync(url).Result; statusCode = response.StatusCode.ToString(); if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } public static string RestfulGet(string url) { HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest; // Get response using (HttpWebResponse response = request.GetResponse() as HttpWebResponse) { // Get the response stream StreamReader reader = new StreamReader(response.GetResponseStream()); // Console application output return reader.ReadToEnd(); } } public static T GetResponse(string url) where T : class, new() { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json")); HttpResponseMessage response = httpClient.GetAsync(url).Result; T result = default(T); if (response.IsSuccessStatusCode) { Task<string> t = response.Content.ReadAsStringAsync(); string s = t.Result; result = JsonConvert.DeserializeObject(s); } return result; } /// /// post請求 /// /// /// post數(shù)據(jù) /// public static string PostResponse(string url, string postData, out string statusCode) { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); httpContent.Headers.ContentType.CharSet = "utf-8"; HttpClient httpClient = new HttpClient(); //httpClient..setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8"); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; statusCode = response.StatusCode.ToString(); if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 發(fā)起post請求 /// /// /// url /// post數(shù)據(jù) /// public static T PostResponse(string url, string postData) where T : class, new() { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); HttpClient httpClient = new HttpClient(); T result = default(T); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; if (response.IsSuccessStatusCode) { Task<string> t = response.Content.ReadAsStringAsync(); string s = t.Result; result = JsonConvert.DeserializeObject(s); } return result; } /// /// 反序列化Xml /// /// /// /// public static T XmlDeserialize(string xmlString) where T : class, new() { try { XmlSerializer ser = new XmlSerializer(typeof(T)); using (StringReader reader = new StringReader(xmlString)) { return (T)ser.Deserialize(reader); } } catch (Exception ex) { throw new Exception("XmlDeserialize發(fā)生異常:xmlString:" + xmlString + "異常信息:" + ex.Message); } } public static string PostResponse(string url, string postData, string token, string appId, string serviceURL, out string statusCode) { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); httpContent.Headers.ContentType.CharSet = "utf-8"; httpContent.Headers.Add("token", token); httpContent.Headers.Add("appId", appId); httpContent.Headers.Add("serviceURL", serviceURL); HttpClient httpClient = new HttpClient(); //httpClient..setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8"); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; statusCode = response.StatusCode.ToString(); if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 修改API /// /// /// /// public static string KongPatchResponse(string url, string postData) { var httpWebRequest = (HttpWebRequest)WebRequest.Create(url); httpWebRequest.ContentType = "application/x-www-form-urlencoded"; httpWebRequest.Method = "PATCH"; byte[] btBodys = Encoding.UTF8.GetBytes(postData); httpWebRequest.ContentLength = btBodys.Length; httpWebRequest.GetRequestStream().Write(btBodys, 0, btBodys.Length); HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse(); var streamReader = new StreamReader(httpWebResponse.GetResponseStream()); string responseContent = streamReader.ReadToEnd(); httpWebResponse.Close(); streamReader.Close(); httpWebRequest.Abort(); httpWebResponse.Close(); return responseContent; } /// /// 創(chuàng)建API /// /// /// /// public static string KongAddResponse(string url, string postData) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded") { CharSet = "utf-8" }; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 刪除API /// /// /// public static bool KongDeleteResponse(string url) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.DeleteAsync(url).Result; return response.IsSuccessStatusCode; } /// /// 修改或者更改API? ? /// /// /// /// public static string KongPutResponse(string url, string postData) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded") { CharSet = "utf-8" }; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.PutAsync(url, httpContent).Result; if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 檢索API /// /// /// public static string KongSerchResponse(string url) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.GetAsync(url).Result; if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } }
到此這篇關于C#使用HttpClient獲取HTTP請求與響應的文章就介紹到這了。希望對大家的學習有所幫助,也希望大家多多支持腳本之家。
相關文章
C#使用linq語句查詢數(shù)組中以特定字符開頭元素的方法
這篇文章主要介紹了C#使用linq語句查詢數(shù)組中以特定字符開頭元素的方法,涉及C#使用linq進行查詢的技巧,具有一定參考借鑒價值,需要的朋友可以參考下2015-04-04C#的靜態(tài)工廠方法與構造函數(shù)相比有哪些優(yōu)缺點
這篇文章主要介紹了C#的靜態(tài)工廠方法與構造函數(shù)對比的優(yōu)缺點,文中示例代碼非常詳細,幫助大家更好的理解和學習,感興趣的朋友可以了解下2020-07-07C#中的ICustomFormatter及IFormatProvider接口用法揭秘
這篇文章主要介紹了C#中的ICustomFormatter及IFormatProvider接口用法揭秘,本文能過分析一段代碼得出一些研究結果,需要的朋友可以參考下2015-06-06