プログラム言語 HTTPクライアント

GET

using System.Net;
using System.IO;

リクエスト作æˆ
var req = WebRequest.Create(@"http://office-yone.com");
リクエストé€ä¿¡ï¼†ãƒ¬ã‚¹ãƒãƒ³ã‚¹å–å¾—
var res = req.GetResponse();
↑ å—信データã‹ã‚‰Streamã‚’å–å¾—
Stream resStream = res.GetResponseStream();
読ã¿è¾¼ã¿
var reader = new StreamReader(st, Encoding.GetEncoding("Shift_JIS"));
string html = reader.ReadToEnd();
reader.Close();
st.Close();

$ curl http://~.com
$conn = curl_init();

å–å¾—ã™ã‚‹URLを指定
$url = "http://office-yone.com/";
curl_setopt ( $conn, CURLOPT_URL , $url );
$response = curl_exec ( $conn );
エラー内容
echo curl_errno($conn);
echo curl_error($conn);
curl_close($conn);
echo('<pre>');
print_r($response);
echo('</pre>');

use Cake\Http\Client;

$http = new Client();
$response = $http->get('https://office-yone.com/');

クエリパラメータ
$response = $http->get('https://office-yone.com/',
 ['prm1' => 'aaa', 'prm2' => 'bbb']
);

echo '<pre>';
print_r($response);
echo '</pre>';
exit;

import requests as req

try:
  
  res = req.get('https://automatetheboringstuff.com/files/rj.txt')
  
  type(res)
  <class 'requests.models.Response'>
  
  res.text[0:250]
  The Project Gutenberg EBook of Romeo and Juliet, by William Shakespeare ~
  
  res.status_code
  200
  
  res.status_code==req.codes.ok
  True
  
  レスãƒãƒ³ã‚¹ã‚¨ãƒ©ãƒ¼ãƒã‚§ãƒƒã‚¯
  res.raise_for_status()
  OKã®å ´åˆä½•ã‚‚èµ·ã“らãªã„
  
  ãƒã‚¤ãƒŠãƒªãƒ¢ãƒ¼ãƒ‰ã§é–‹ã(UNICODE文字を維æŒã™ã‚‹ç‚º)
  file = open('result.txt', 'wb')
  
  100KBãšã¤ãƒã‚¤ãƒŠãƒªãƒ‡ãƒ¼ã‚¿ã‚’å–å¾—
  for chunk in res.iter_content(100000):
    書込
    file.write(chunk)
  file.close()
  
  res = req.get('https://automatetheboringstuff.com/files/aaa.txt')
  存在ã—ãªã„URL
  
  レスãƒãƒ³ã‚¹ã‚¨ãƒ©ãƒ¼ãƒã‚§ãƒƒã‚¯
  res.raise_for_status()
  NGã®å ´åˆHTTPError例外ãŒç™ºç”Ÿã™ã‚‹
  
except Exception as err:
  print(str(err))
  404 Client Error: Not Found for url: https://automatetheboringstuff.com/files/aaa.txt

POST

using System.Net;
using System.IO;

WebRequestã®ä½œæˆ
var req = WebRequest.Create("http://~;");
POSTé€ä¿¡ã™ã‚‹æ–‡å­—列をãƒã‚¤ãƒˆåž‹é…列ã«å¤‰æ›
byte[] postDataBytes = Encoding.ASCII.GetBytes("AAA");
メソッドã«POSTを指定
req.Method = "POST";
ContentTypeã‚’"application/x-www-form-urlencoded"ã«
req.ContentType = "application/x-www-form-urlencoded";
POSTé€ä¿¡ã™ã‚‹ãƒ‡ãƒ¼ã‚¿ã®é•·ã•を指定
req.ContentLength = postDataBytes.Length;
POSTé€ä¿¡ã®ç‚ºã®Streamã‚’å–å¾—
Stream reqStream = req.GetRequestStream();
é€ä¿¡ã™ã‚‹ãƒ‡ãƒ¼ã‚¿æ›¸ãè¾¼ã¿
reqStream.Write(postDataBytes, 0, postDataBytes.Length);
reqStream.Close();
リクエストメッセージå–å¾—
var res = req.GetResponse();
↑ å—信データã‹ã‚‰Streamã‚’å–å¾—
Stream resStream = res.GetResponseStream();
読ã¿è¾¼ã¿
StreamReader sr = new StreamReader(resStream, Encoding.GetEncoding("shift_jis"));
string html = reader.ReadToEnd();
reader.Close();
st.Close();

$ curl
-w '\n' 'http://~.com'
-d 'prm1=aaaa&prm2=bbb&prm3=ccc'
-XPOST
$conn = curl_init();

// å–å¾—ã™ã‚‹URLを指定
$url = "http://office-yone.com/";
curl_setopt ( $conn, CURLOPT_URL , $url );

curl_setopt($ch, CURLOPT_POST, TRUE);
パラメータ
curl_setopt($ch, CURLOPT_POSTFIELDS, "prm1=aaa&prm2=bbb");

$response = curl_exec ( $conn );
curl_close($conn);

use Cake\Http\Client;

$http = new Client();

ãƒã‚¹ãƒˆãƒ‘ラメータ
$response = $http->post('https://office-yone.com/',
 ['prm1' => 'aaa', 'prm2' => 'bbb']
);

echo '<pre>';
print_r($response);
echo '</pre>';
exit;

HTMLå–å¾—

using System.Net;
using System.IO;
指定ã®URLデータをå–å¾—
Stream st = new WebClient().OpenRead("http://office-yone.com");
var sr = new StreamReader(st, Encoding.GetEncoding("Shift_JIS"));
string html = sr.ReadToEnd();
sr.Close();
st.Close();
MessageBox.Show(html);

$conn = curl_init();

å–å¾—ã™ã‚‹URLを指定
$url = "http://office-yone.com/";
curl_setopt ( $conn, CURLOPT_URL , $url );

å®Ÿè¡Œçµæžœã‚’文字列ã§è¿”ã™
curl_setopt($conn, CURLOPT_RETURNTRANSFER, 1);
$response = curl_exec ( $conn );
curl_close($conn);

クッキーé€å—ä¿¡

DownloadFile

using System.Net;
指定ã®URLデータをå–得&ä¿å­˜
new WebClient().DownloadFile(address: @"http://office-yone.com", fileName: @"office-yone.html");

$ curl http://~.com -o

エラーメッセージ表示
$ curl http://~.com -o -s

DownloadData

using System.Net;
指定ã®URLデータをå–å¾—
byte[] data = new WebClient().DownloadData("http://office-yone.com");
string html = Encoding.GetEncoding("Shift_JIS").GetString(data);
MessageBox.Show(html);

プロキシ利用

using System.Net;
var client = new WebClient();
プロキシ利用
client.Proxy = WebRequest.DefaultWebProxy;
プロキシ指定
client.Proxy = new WebProxy("http:~:8080");
プロキシ未使用
client.Proxy = null;
client.~

$conn = curl_init();

// å–å¾—ã™ã‚‹URLを指定
$url = "http://office-yone.com/";
curl_setopt ( $conn, CURLOPT_URL , $url );

curl_setopt ($conn , CURLOPT_PROXY, '999.999.999.999.9999);

$response = curl_exec ( $conn );
curl_close($conn);

セキュリティ

$conn = curl_init();

// å–å¾—ã™ã‚‹URLを指定
$url = "http://office-yone.com/";
curl_setopt ( $conn, CURLOPT_URL , $url );

è©³ç´°ãªæƒ…報を出力。STDERRã‹ã€ã¾ãŸã¯CURLOPT_STDERRã§æŒ‡å®šã—ãŸãƒ•ァイルã«å‡ºåŠ›ã•れる
curl_setopt($ch, CURLOPT_VERBOSE, TRUE);

サーãƒãƒ¼è¨¼æ˜Žæ›¸ã®æ¤œè¨¼ã‚’行ã‚ãªã„
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);

1:SSLピア証明書ã«ä¸€èˆ¬åãŒå­˜åœ¨ã™ã‚‹ã‹ã©ã†ã‹ã‚’調ã¹ã‚‹
2:1ã«åŠ ãˆã€ãã®åå‰ãŒãƒ›ã‚¹ãƒˆåã¨ä¸€è‡´ã™ã‚‹ã“ã¨ã‚’検証ã™ã‚‹â€»è¦å®šå€¤
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);

å–å¾—çµæžœã‚’加工ã—ãªã„
curl_setopt($conn, CURLOPT_RETURNTRANSFER, TRUE);

$response = curl_exec ( $conn );
curl_close($conn);

ブラウザ利用

import webbrowser as brows

ãƒ–ãƒ©ã‚¦ã‚¶ã§æŒ‡å®šURLã‚’é–‹ã
brows.open('https://office-yone.com/web_access')

Webスクレイピング

import requests as req
import bs4

res = req.get('https://office-yone.com/web_access/')

soup = bs4.BeautifulSoup(res.text)

type(soup)
<class 'bs4.BeautifulSoup'>

elm = soup.select('h5')
type(elm)
<class 'list'>

for item in elm:
  item
  <h5>GET</h5>
  <h5>POST</h5>

Follow me!