1 // HTTP utils based on https://github.com/adamdruppe/arsd/blob/master/http.d
2 // copyright Adam D. Ruppe
3 module HttpClient;
4 
5 import std.socket;
6 import std..string;
7 import std.conv;
8 static import std.algorithm;
9 static import std.uri;
10 
11 ubyte[] getBinary(string url, string[string] cookies = null) {
12 	auto hr = httpRequest("GET", url, null, cookies);
13 
14 	const uint maxAttempts = 32;
15 	uint attempts = 0;
16 
17 	while (hr.code == 302 && attempts < maxAttempts) {
18 		hr = httpRequest("GET", hr.headers["Location"], null, cookies);
19 		++attempts;
20 	}
21 
22 	if(hr.code != 200)
23 		throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url));
24 
25 	return hr.content;
26 }
27 
28 struct HttpResponse {
29 	int code;
30 	string contentType;
31 	string[string] cookies;
32 	string[string] headers;
33 	ubyte[] content;
34 }
35 
36 struct UriParts {
37 	string original;
38 	string method;
39 	string host;
40 	ushort port;
41 	string path;
42 
43 	bool useHttps;
44 
45 	this(string uri) {
46 		original = uri;
47 
48 		if(uri[0 .. 8] == "https://")
49 			useHttps = true;
50 		else
51 		if(uri[0..7] != "http://")
52 			throw new Exception("You must use an absolute, http or https URL.");
53 
54 		version(with_openssl) {} else
55 		if(useHttps)
56 			throw new Exception("openssl support not compiled in try -version=with_openssl");
57 
58 		int start = useHttps ? 8 : 7;
59 
60 		auto posSlash = uri[start..$].indexOf("/");
61 		if(posSlash != -1)
62 			posSlash += start;
63 
64 		if(posSlash == -1)
65 			posSlash = uri.length;
66 
67 		auto posColon = uri[start..$].indexOf(":");
68 		if(posColon != -1)
69 			posColon += start;
70 
71 		if(useHttps)
72 			port = 443;
73 		else
74 			port = 80;
75 
76 		if(posColon != -1 && posColon < posSlash) {
77 			host = uri[start..posColon];
78 			port = to!ushort(uri[posColon+1..posSlash]);
79 		} else
80 			host = uri[start..posSlash];
81 
82 		path = uri[posSlash..$];
83 		if(path == "")
84 			path = "/";
85 	}
86 }
87 
88 HttpResponse httpRequest(string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null) {
89 	import std.socket;
90 
91 	auto u = UriParts(uri);
92 	// auto f = openNetwork(u.host, u.port);
93 	auto f = new TcpSocket();
94 	f.connect(new InternetAddress(u.host, u.port));
95 
96 	void delegate(string) write = (string d) {
97 		f.send(d);
98 	};
99 
100 	char[4096] readBuffer; // rawRead actually blocks until it can fill up the whole buffer... which is broken as far as http goes so one char at a time i guess. slow lol
101 	char[] delegate() read = () {
102 		size_t num = f.receive(readBuffer);
103 		return readBuffer[0..num];
104 	};
105 
106 	version(with_openssl) {
107 		import deimos.openssl.ssl;
108 		SSL* ssl;
109 		SSL_CTX* ctx;
110 		if(u.useHttps) {
111 			void sslAssert(bool ret){
112 				if (!ret){
113 					throw new Exception("SSL_ERROR");
114 				}
115 			}
116 			SSL_library_init();
117 			OpenSSL_add_all_algorithms();
118 			SSL_load_error_strings();
119 
120 			ctx = SSL_CTX_new(SSLv3_client_method());
121 			sslAssert(!(ctx is null));
122 
123 			ssl = SSL_new(ctx);
124 			SSL_set_fd(ssl, f.handle);
125 			sslAssert(SSL_connect(ssl) != -1);
126 
127 			write = (string d) {
128 				SSL_write(ssl, d.ptr, cast(uint)d.length);
129 			};
130 
131 			read = () {
132 				auto len = SSL_read(ssl, readBuffer.ptr, readBuffer.length);
133 				return readBuffer[0 .. len];
134 			};
135 		}
136 	}
137 
138 
139 	HttpResponse response = doHttpRequestOnHelpers(write, read, method, uri, content, cookies, headers, u.useHttps);
140 
141 	version(with_openssl) {
142 		if(u.useHttps) {
143 			SSL_free(ssl);
144 			SSL_CTX_free(ctx);
145 		}
146 	}
147 
148 	return response;
149 }
150 
151 /**
152 	Executes a generic http request, returning the full result. The correct formatting
153 	of the parameters are the caller's responsibility. Content-Length is added automatically,
154 	but YOU must give Content-Type!
155 */
156 HttpResponse doHttpRequestOnHelpers(void delegate(string) write, char[] delegate() read, string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null, bool https = false)
157 	in {
158 		assert(method == "POST" || method == "GET");
159 	}
160 body {
161 	auto u = UriParts(uri);
162 
163 
164 
165 
166 
167 	write(format("%s %s HTTP/1.1\r\n", method, u.path));
168 	write(format("Host: %s\r\n", u.host));
169 	write(format("Connection: close\r\n"));
170 	if(content !is null)
171 		write(format("Content-Length: %d\r\n", content.length));
172 
173 	if(cookies !is null) {
174 		string cookieHeader = "Cookie: ";
175 		bool first = true;
176 		foreach(k, v; cookies) {
177 			if(first)
178 				first = false;
179 			else
180 				cookieHeader ~= "; ";
181 			cookieHeader ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
182 		}
183 
184 		write(format("%s\r\n", cookieHeader));
185 	}
186 
187 	if(headers !is null)
188 		foreach(header; headers)
189 			write(format("%s\r\n", header));
190 	write("\r\n");
191 	if(content !is null)
192 		write(cast(string) content);
193 
194 
195 	string buffer;
196 
197 	string readln() {
198 		auto idx = buffer.indexOf("\r\n");
199 		if(idx == -1) {
200 			auto more = read();
201 			if(more.length == 0) { // end of file or something
202 				auto ret = buffer;
203 				buffer = null;
204 				return ret;
205 			}
206 			buffer ~= more;
207 			return readln();
208 		}
209 		auto ret = buffer[0 .. idx + 2]; // + the \r\n
210 		if(idx + 2 < buffer.length)
211 			buffer = buffer[idx + 2 .. $];
212 		else
213 			buffer = null;
214 		return ret;
215 	}
216 
217 	HttpResponse hr;
218  cont:
219 	string l = readln();
220 	if(l[0..9] != "HTTP/1.1 ")
221 		throw new Exception("Not talking to a http server");
222 
223 	hr.code = to!int(l[9..12]); // HTTP/1.1 ### OK
224 
225 	if(hr.code == 100) { // continue
226 		do {
227 			l = readln();
228 		} while(l.length > 1);
229 
230 		goto cont;
231 	}
232 
233 	bool chunked = false;
234 
235 	auto line = readln();
236 	while(line.length) {
237 		if(line.strip.length == 0)
238 			break;
239 
240 		import std.algorithm : findSplit;
241 		import std.range : empty;
242 		auto split = findSplit(line, ":");
243 
244 		if (!split[0].empty && !split[1].empty)
245 			hr.headers[split[0]] = split[2].strip();
246 
247 		if(line.startsWith("Content-Type: "))
248 			hr.contentType = line[14..$-1];
249 		if(line.startsWith("Set-Cookie: ")) {
250 			auto hdr = line["Set-Cookie: ".length .. $-1];
251 			auto semi = hdr.indexOf(";");
252 			if(semi != -1)
253 				hdr = hdr[0 .. semi];
254 
255 			auto equal = hdr.indexOf("=");
256 			string name, value;
257 			if(equal == -1) {
258 				name = hdr;
259 				// doesn't this mean erase the cookie?
260 			} else {
261 				name = hdr[0 .. equal];
262 				value = hdr[equal + 1 .. $];
263 			}
264 
265 			name = std.uri.decodeComponent(name);
266 			value = std.uri.decodeComponent(value);
267 
268 			hr.cookies[name] = value;
269 		}
270 		if(line.startsWith("Transfer-Encoding: chunked"))
271 			chunked = true;
272 		line = readln();
273 	}
274 
275 	// there might be leftover stuff in the line buffer
276 	ubyte[] response = cast(ubyte[]) buffer.dup;
277 	auto part = read();
278 	while(part.length) {
279 		response ~= part;
280 		part = read();
281 	}
282 
283 	if(chunked) {
284 		// read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character
285 		// read binary data of that length. it is our content
286 		// repeat until a zero sized chunk
287 		// then read footers as headers.
288 
289 		int state = 0;
290 		int size;
291 		int start = 0;
292 		for(int a = 0; a < response.length; a++) {
293 			final switch(state) {
294 				case 0: // reading hex
295 					char c = response[a];
296 					if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
297 						// just keep reading
298 					} else {
299 						int power = 1;
300 						size = 0;
301 						for(int b = a-1; b >= start; b--) {
302 							char cc = response[b];
303 							if(cc >= 'a' && cc <= 'z')
304 								cc -= 0x20;
305 							int val = 0;
306 							if(cc >= '0' && cc <= '9')
307 								val = cc - '0';
308 							else
309 								val = cc - 'A' + 10;
310 
311 							size += power * val;
312 							power *= 16;
313 						}
314 						state++;
315 						continue;
316 					}
317 				break;
318 				case 1: // reading until end of line
319 					char c = response[a];
320 					if(c == '\n') {
321 						if(size == 0)
322 							state = 3;
323 						else
324 							state = 2;
325 					}
326 				break;
327 				case 2: // reading data
328 					hr.content ~= response[a..a+size];
329 					a += size;
330 					a+= 1; // skipping a 13 10
331 					start = a + 1;
332 					state = 0;
333 				break;
334 				case 3: // reading footers
335 					goto done; // FIXME
336 			}
337 		}
338 	} else
339 		hr.content = response;
340 	done:
341 
342 	return hr;
343 }