basic_parser.hpp 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699
  1. //
  2. // Copyright (c) 2016-2019 Vinnie Falco (vinnie dot falco at gmail dot com)
  3. //
  4. // Distributed under the Boost Software License, Version 1.0. (See accompanying
  5. // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  6. //
  7. // Official repository: https://github.com/boostorg/beast
  8. //
  9. #ifndef BOOST_BEAST_HTTP_BASIC_PARSER_HPP
  10. #define BOOST_BEAST_HTTP_BASIC_PARSER_HPP
  11. #include <boost/beast/core/detail/config.hpp>
  12. #include <boost/beast/core/error.hpp>
  13. #include <boost/beast/core/string.hpp>
  14. #include <boost/beast/http/field.hpp>
  15. #include <boost/beast/http/verb.hpp>
  16. #include <boost/beast/http/detail/basic_parser.hpp>
  17. #include <boost/asio/buffer.hpp>
  18. #include <boost/optional.hpp>
  19. #include <boost/assert.hpp>
  20. #include <limits>
  21. #include <memory>
  22. #include <type_traits>
  23. #include <utility>
  24. namespace boost {
  25. namespace beast {
  26. namespace http {
  27. /** A parser for decoding HTTP/1 wire format messages.
  28. This parser is designed to efficiently parse messages in the
  29. HTTP/1 wire format. It allocates no memory when input is
  30. presented as a single contiguous buffer, and uses minimal
  31. state. It will handle chunked encoding and it understands
  32. the semantics of the Connection, Content-Length, and Upgrade
  33. fields.
  34. The parser is optimized for the case where the input buffer
  35. sequence consists of a single contiguous buffer. The
  36. @ref beast::basic_flat_buffer class is provided, which guarantees
  37. that the input sequence of the stream buffer will be represented
  38. by exactly one contiguous buffer. To ensure the optimum performance
  39. of the parser, use @ref beast::basic_flat_buffer with HTTP algorithms
  40. such as @ref read, @ref read_some, @ref async_read, and @ref async_read_some.
  41. Alternatively, the caller may use custom techniques to ensure that
  42. the structured portion of the HTTP message (header or chunk header)
  43. is contained in a linear buffer.
  44. The interface to the parser uses virtual member functions.
  45. To use this class, derive your type from @ref basic_parser. When
  46. bytes are presented, the implementation will make a series of zero
  47. or more calls to virtual functions, which the derived class must
  48. implement.
  49. Every virtual function must be provided by the derived class,
  50. or else a compilation error will be generated. The implementation
  51. will make sure that `ec` is clear before each virtual function
  52. is invoked. If a virtual function sets an error, it is propagated
  53. out of the parser to the caller.
  54. @tparam isRequest A `bool` indicating whether the parser will be
  55. presented with request or response message.
  56. @note If the parser encounters a field value with obs-fold
  57. longer than 4 kilobytes in length, an error is generated.
  58. */
  59. template<bool isRequest>
  60. class basic_parser
  61. : private detail::basic_parser_base
  62. {
  63. boost::optional<std::uint64_t>
  64. body_limit_ =
  65. boost::optional<std::uint64_t>(
  66. default_body_limit(is_request{})); // max payload body
  67. std::uint64_t len_ = 0; // size of chunk or body
  68. std::uint64_t len0_ = 0; // content length if known
  69. std::unique_ptr<char[]> buf_; // temp storage
  70. std::size_t buf_len_ = 0; // size of buf_
  71. std::size_t skip_ = 0; // resume search here
  72. std::uint32_t header_limit_ = 8192; // max header size
  73. unsigned short status_ = 0; // response status
  74. state state_ = state::nothing_yet; // initial state
  75. unsigned f_ = 0; // flags
  76. // limit on the size of the stack flat buffer
  77. static std::size_t constexpr max_stack_buffer = 8192;
  78. // Message will be complete after reading header
  79. static unsigned constexpr flagSkipBody = 1<< 0;
  80. // Consume input buffers across semantic boundaries
  81. static unsigned constexpr flagEager = 1<< 1;
  82. // The parser has read at least one byte
  83. static unsigned constexpr flagGotSome = 1<< 2;
  84. // Message semantics indicate a body is expected.
  85. // cleared if flagSkipBody set
  86. //
  87. static unsigned constexpr flagHasBody = 1<< 3;
  88. static unsigned constexpr flagHTTP11 = 1<< 4;
  89. static unsigned constexpr flagNeedEOF = 1<< 5;
  90. static unsigned constexpr flagExpectCRLF = 1<< 6;
  91. static unsigned constexpr flagConnectionClose = 1<< 7;
  92. static unsigned constexpr flagConnectionUpgrade = 1<< 8;
  93. static unsigned constexpr flagConnectionKeepAlive = 1<< 9;
  94. static unsigned constexpr flagContentLength = 1<< 10;
  95. static unsigned constexpr flagChunked = 1<< 11;
  96. static unsigned constexpr flagUpgrade = 1<< 12;
  97. static unsigned constexpr flagFinalChunk = 1<< 13;
  98. static constexpr
  99. std::uint64_t
  100. default_body_limit(std::true_type)
  101. {
  102. // limit for requests
  103. return 1 * 1024 * 1024; // 1MB
  104. }
  105. static constexpr
  106. std::uint64_t
  107. default_body_limit(std::false_type)
  108. {
  109. // limit for responses
  110. return 8 * 1024 * 1024; // 8MB
  111. }
  112. template<bool OtherIsRequest>
  113. friend class basic_parser;
  114. friend class basic_parser_test;
  115. protected:
  116. /// Default constructor
  117. basic_parser() = default;
  118. /** Move constructor
  119. @note
  120. After the move, the only valid operation on the
  121. moved-from object is destruction.
  122. */
  123. basic_parser(basic_parser &&) = default;
  124. /// Move assignment
  125. basic_parser& operator=(basic_parser &&) = default;
  126. public:
  127. /// `true` if this parser parses requests, `false` for responses.
  128. using is_request =
  129. std::integral_constant<bool, isRequest>;
  130. /// Destructor
  131. virtual ~basic_parser() = default;
  132. /// Copy constructor
  133. basic_parser(basic_parser const&) = delete;
  134. /// Copy assignment
  135. basic_parser& operator=(basic_parser const&) = delete;
  136. /// Returns `true` if the parser has received at least one byte of input.
  137. bool
  138. got_some() const
  139. {
  140. return state_ != state::nothing_yet;
  141. }
  142. /** Returns `true` if the message is complete.
  143. The message is complete after the full header is prduced
  144. and one of the following is true:
  145. @li The skip body option was set.
  146. @li The semantics of the message indicate there is no body.
  147. @li The semantics of the message indicate a body is expected,
  148. and the entire body was parsed.
  149. */
  150. bool
  151. is_done() const
  152. {
  153. return state_ == state::complete;
  154. }
  155. /** Returns `true` if a the parser has produced the full header.
  156. */
  157. bool
  158. is_header_done() const
  159. {
  160. return state_ > state::fields;
  161. }
  162. /** Returns `true` if the message is an upgrade message.
  163. @note The return value is undefined unless
  164. @ref is_header_done would return `true`.
  165. */
  166. bool
  167. upgrade() const
  168. {
  169. return (f_ & flagConnectionUpgrade) != 0;
  170. }
  171. /** Returns `true` if the last value for Transfer-Encoding is "chunked".
  172. @note The return value is undefined unless
  173. @ref is_header_done would return `true`.
  174. */
  175. bool
  176. chunked() const
  177. {
  178. return (f_ & flagChunked) != 0;
  179. }
  180. /** Returns `true` if the message has keep-alive connection semantics.
  181. This function always returns `false` if @ref need_eof would return
  182. `false`.
  183. @note The return value is undefined unless
  184. @ref is_header_done would return `true`.
  185. */
  186. bool
  187. keep_alive() const;
  188. /** Returns the optional value of Content-Length if known.
  189. @note The return value is undefined unless
  190. @ref is_header_done would return `true`.
  191. */
  192. boost::optional<std::uint64_t>
  193. content_length() const;
  194. /** Returns the remaining content length if known
  195. If the message header specifies a Content-Length,
  196. the return value will be the number of bytes remaining
  197. in the payload body have not yet been parsed.
  198. @note The return value is undefined unless
  199. @ref is_header_done would return `true`.
  200. */
  201. boost::optional<std::uint64_t>
  202. content_length_remaining() const;
  203. /** Returns `true` if the message semantics require an end of file.
  204. Depending on the contents of the header, the parser may
  205. require and end of file notification to know where the end
  206. of the body lies. If this function returns `true` it will be
  207. necessary to call @ref put_eof when there will never be additional
  208. data from the input.
  209. */
  210. bool
  211. need_eof() const
  212. {
  213. return (f_ & flagNeedEOF) != 0;
  214. }
  215. /** Set the limit on the payload body.
  216. This function sets the maximum allowed size of the payload body,
  217. before any encodings except chunked have been removed. Depending
  218. on the message semantics, one of these cases will apply:
  219. @li The Content-Length is specified and exceeds the limit. In
  220. this case the result @ref error::body_limit is returned
  221. immediately after the header is parsed.
  222. @li The Content-Length is unspecified and the chunked encoding
  223. is not specified as the last encoding. In this case the end of
  224. message is determined by the end of file indicator on the
  225. associated stream or input source. If a sufficient number of
  226. body payload octets are presented to the parser to exceed the
  227. configured limit, the parse fails with the result
  228. @ref error::body_limit
  229. @li The Transfer-Encoding specifies the chunked encoding as the
  230. last encoding. In this case, when the number of payload body
  231. octets produced by removing the chunked encoding exceeds
  232. the configured limit, the parse fails with the result
  233. @ref error::body_limit.
  234. Setting the limit after any body octets have been parsed
  235. results in undefined behavior.
  236. The default limit is 1MB for requests and 8MB for responses.
  237. @param v An optional integral value representing the body limit.
  238. If this is equal to `boost::none`, then the body limit is disabled.
  239. */
  240. void
  241. body_limit(boost::optional<std::uint64_t> v)
  242. {
  243. body_limit_ = v;
  244. }
  245. /** Set a limit on the total size of the header.
  246. This function sets the maximum allowed size of the header
  247. including all field name, value, and delimiter characters
  248. and also including the CRLF sequences in the serialized
  249. input. If the end of the header is not found within the
  250. limit of the header size, the error @ref error::header_limit
  251. is returned by @ref put.
  252. Setting the limit after any header octets have been parsed
  253. results in undefined behavior.
  254. */
  255. void
  256. header_limit(std::uint32_t v)
  257. {
  258. header_limit_ = v;
  259. }
  260. /// Returns `true` if the eager parse option is set.
  261. bool
  262. eager() const
  263. {
  264. return (f_ & flagEager) != 0;
  265. }
  266. /** Set the eager parse option.
  267. Normally the parser returns after successfully parsing a structured
  268. element (header, chunk header, or chunk body) even if there are octets
  269. remaining in the input. This is necessary when attempting to parse the
  270. header first, or when the caller wants to inspect information which may
  271. be invalidated by subsequent parsing, such as a chunk extension. The
  272. `eager` option controls whether the parser keeps going after parsing
  273. structured element if there are octets remaining in the buffer and no
  274. error occurs. This option is automatically set or cleared during certain
  275. stream operations to improve performance with no change in functionality.
  276. The default setting is `false`.
  277. @param v `true` to set the eager parse option or `false` to disable it.
  278. */
  279. void
  280. eager(bool v)
  281. {
  282. if(v)
  283. f_ |= flagEager;
  284. else
  285. f_ &= ~flagEager;
  286. }
  287. /// Returns `true` if the skip parse option is set.
  288. bool
  289. skip() const
  290. {
  291. return (f_ & flagSkipBody) != 0;
  292. }
  293. /** Set the skip parse option.
  294. This option controls whether or not the parser expects to see an HTTP
  295. body, regardless of the presence or absence of certain fields such as
  296. Content-Length or a chunked Transfer-Encoding. Depending on the request,
  297. some responses do not carry a body. For example, a 200 response to a
  298. CONNECT request from a tunneling proxy, or a response to a HEAD request.
  299. In these cases, callers may use this function inform the parser that
  300. no body is expected. The parser will consider the message complete
  301. after the header has been received.
  302. @param v `true` to set the skip body option or `false` to disable it.
  303. @note This function must called before any bytes are processed.
  304. */
  305. void
  306. skip(bool v);
  307. /** Write a buffer sequence to the parser.
  308. This function attempts to incrementally parse the HTTP
  309. message data stored in the caller provided buffers. Upon
  310. success, a positive return value indicates that the parser
  311. made forward progress, consuming that number of
  312. bytes.
  313. In some cases there may be an insufficient number of octets
  314. in the input buffer in order to make forward progress. This
  315. is indicated by the code @ref error::need_more. When
  316. this happens, the caller should place additional bytes into
  317. the buffer sequence and call @ref put again.
  318. The error code @ref error::need_more is special. When this
  319. error is returned, a subsequent call to @ref put may succeed
  320. if the buffers have been updated. Otherwise, upon error
  321. the parser may not be restarted.
  322. @param buffers An object meeting the requirements of
  323. <em>ConstBufferSequence</em> that represents the next chunk of
  324. message data. If the length of this buffer sequence is
  325. one, the implementation will not allocate additional memory.
  326. The class @ref beast::basic_flat_buffer is provided as one way to
  327. meet this requirement
  328. @param ec Set to the error, if any occurred.
  329. @return The number of octets consumed in the buffer
  330. sequence. The caller should remove these octets even if the
  331. error is set.
  332. */
  333. template<class ConstBufferSequence>
  334. std::size_t
  335. put(ConstBufferSequence const& buffers, error_code& ec);
  336. #if ! BOOST_BEAST_DOXYGEN
  337. std::size_t
  338. put(net::const_buffer buffer,
  339. error_code& ec);
  340. #endif
  341. /** Inform the parser that the end of stream was reached.
  342. In certain cases, HTTP needs to know where the end of
  343. the stream is. For example, sometimes servers send
  344. responses without Content-Length and expect the client
  345. to consume input (for the body) until EOF. Callbacks
  346. and errors will still be processed as usual.
  347. This is typically called when a read from the
  348. underlying stream object sets the error code to
  349. `net::error::eof`.
  350. @note Only valid after parsing a complete header.
  351. @param ec Set to the error, if any occurred.
  352. */
  353. void
  354. put_eof(error_code& ec);
  355. protected:
  356. /** Called after receiving the request-line.
  357. This virtual function is invoked after receiving a request-line
  358. when parsing HTTP requests.
  359. It can only be called when `isRequest == true`.
  360. @param method The verb enumeration. If the method string is not
  361. one of the predefined strings, this value will be @ref verb::unknown.
  362. @param method_str The unmodified string representing the verb.
  363. @param target The request-target.
  364. @param version The HTTP-version. This will be 10 for HTTP/1.0,
  365. and 11 for HTTP/1.1.
  366. @param ec An output parameter which the function may set to indicate
  367. an error. The error will be clear before this function is invoked.
  368. */
  369. virtual
  370. void
  371. on_request_impl(
  372. verb method,
  373. string_view method_str,
  374. string_view target,
  375. int version,
  376. error_code& ec) = 0;
  377. /** Called after receiving the status-line.
  378. This virtual function is invoked after receiving a status-line
  379. when parsing HTTP responses.
  380. It can only be called when `isRequest == false`.
  381. @param code The numeric status code.
  382. @param reason The reason-phrase. Note that this value is
  383. now obsolete, and only provided for historical or diagnostic
  384. purposes.
  385. @param version The HTTP-version. This will be 10 for HTTP/1.0,
  386. and 11 for HTTP/1.1.
  387. @param ec An output parameter which the function may set to indicate
  388. an error. The error will be clear before this function is invoked.
  389. */
  390. virtual
  391. void
  392. on_response_impl(
  393. int code,
  394. string_view reason,
  395. int version,
  396. error_code& ec) = 0;
  397. /** Called once for each complete field in the HTTP header.
  398. This virtual function is invoked for each field that is received
  399. while parsing an HTTP message.
  400. @param name The known field enum value. If the name of the field
  401. is not recognized, this value will be @ref field::unknown.
  402. @param name_string The exact name of the field as received from
  403. the input, represented as a string.
  404. @param value A string holding the value of the field.
  405. @param ec An output parameter which the function may set to indicate
  406. an error. The error will be clear before this function is invoked.
  407. */
  408. virtual
  409. void
  410. on_field_impl(
  411. field name,
  412. string_view name_string,
  413. string_view value,
  414. error_code& ec) = 0;
  415. /** Called once after the complete HTTP header is received.
  416. This virtual function is invoked once, after the complete HTTP
  417. header is received while parsing a message.
  418. @param ec An output parameter which the function may set to indicate
  419. an error. The error will be clear before this function is invoked.
  420. */
  421. virtual
  422. void
  423. on_header_impl(error_code& ec) = 0;
  424. /** Called once before the body is processed.
  425. This virtual function is invoked once, before the content body is
  426. processed (but after the complete header is received).
  427. @param content_length A value representing the content length in
  428. bytes if the length is known (this can include a zero length).
  429. Otherwise, the value will be `boost::none`.
  430. @param ec An output parameter which the function may set to indicate
  431. an error. The error will be clear before this function is invoked.
  432. */
  433. virtual
  434. void
  435. on_body_init_impl(
  436. boost::optional<std::uint64_t> const& content_length,
  437. error_code& ec) = 0;
  438. /** Called each time additional data is received representing the content body.
  439. This virtual function is invoked for each piece of the body which is
  440. received while parsing of a message. This function is only used when
  441. no chunked transfer encoding is present.
  442. @param body A string holding the additional body contents. This may
  443. contain nulls or unprintable characters.
  444. @param ec An output parameter which the function may set to indicate
  445. an error. The error will be clear before this function is invoked.
  446. @see on_chunk_body_impl
  447. */
  448. virtual
  449. std::size_t
  450. on_body_impl(
  451. string_view body,
  452. error_code& ec) = 0;
  453. /** Called each time a new chunk header of a chunk encoded body is received.
  454. This function is invoked each time a new chunk header is received.
  455. The function is only used when the chunked transfer encoding is present.
  456. @param size The size of this chunk, in bytes.
  457. @param extensions A string containing the entire chunk extensions.
  458. This may be empty, indicating no extensions are present.
  459. @param ec An output parameter which the function may set to indicate
  460. an error. The error will be clear before this function is invoked.
  461. */
  462. virtual
  463. void
  464. on_chunk_header_impl(
  465. std::uint64_t size,
  466. string_view extensions,
  467. error_code& ec) = 0;
  468. /** Called each time additional data is received representing part of a body chunk.
  469. This virtual function is invoked for each piece of the body which is
  470. received while parsing of a message. This function is only used when
  471. no chunked transfer encoding is present.
  472. @param remain The number of bytes remaining in this chunk. This includes
  473. the contents of passed `body`. If this value is zero, then this represents
  474. the final chunk.
  475. @param body A string holding the additional body contents. This may
  476. contain nulls or unprintable characters.
  477. @param ec An output parameter which the function may set to indicate
  478. an error. The error will be clear before this function is invoked.
  479. @return This function should return the number of bytes actually consumed
  480. from the `body` value. Any bytes that are not consumed on this call
  481. will be presented in a subsequent call.
  482. @see on_body_impl
  483. */
  484. virtual
  485. std::size_t
  486. on_chunk_body_impl(
  487. std::uint64_t remain,
  488. string_view body,
  489. error_code& ec) = 0;
  490. /** Called once when the complete message is received.
  491. This virtual function is invoked once, after successfully parsing
  492. a complete HTTP message.
  493. @param ec An output parameter which the function may set to indicate
  494. an error. The error will be clear before this function is invoked.
  495. */
  496. virtual
  497. void
  498. on_finish_impl(error_code& ec) = 0;
  499. private:
  500. boost::optional<std::uint64_t>
  501. content_length_unchecked() const;
  502. template<class ConstBufferSequence>
  503. std::size_t
  504. put_from_stack(
  505. std::size_t size,
  506. ConstBufferSequence const& buffers,
  507. error_code& ec);
  508. void
  509. maybe_need_more(
  510. char const* p, std::size_t n,
  511. error_code& ec);
  512. void
  513. parse_start_line(
  514. char const*& p, char const* last,
  515. error_code& ec, std::true_type);
  516. void
  517. parse_start_line(
  518. char const*& p, char const* last,
  519. error_code& ec, std::false_type);
  520. void
  521. parse_fields(
  522. char const*& p, char const* last,
  523. error_code& ec);
  524. void
  525. finish_header(
  526. error_code& ec, std::true_type);
  527. void
  528. finish_header(
  529. error_code& ec, std::false_type);
  530. void
  531. parse_body(char const*& p,
  532. std::size_t n, error_code& ec);
  533. void
  534. parse_body_to_eof(char const*& p,
  535. std::size_t n, error_code& ec);
  536. void
  537. parse_chunk_header(char const*& p,
  538. std::size_t n, error_code& ec);
  539. void
  540. parse_chunk_body(char const*& p,
  541. std::size_t n, error_code& ec);
  542. void
  543. do_field(field f,
  544. string_view value, error_code& ec);
  545. };
  546. } // http
  547. } // beast
  548. } // boost
  549. #include <boost/beast/http/impl/basic_parser.hpp>
  550. #ifdef BOOST_BEAST_HEADER_ONLY
  551. #include <boost/beast/http/impl/basic_parser.ipp>
  552. #endif
  553. #endif