Libosmium  2.11.1
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
reader.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_READER_HPP
2 #define OSMIUM_IO_READER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cerrno>
37 #include <cstdlib>
38 #include <fcntl.h>
39 #include <future>
40 #include <memory>
41 #include <string>
42 #include <system_error>
43 #include <thread>
44 #include <utility>
45 
46 #ifndef _WIN32
47 # include <sys/wait.h>
48 #endif
49 
50 #ifndef _MSC_VER
51 # include <unistd.h>
52 #endif
53 
55 #include <osmium/io/detail/input_format.hpp>
56 #include <osmium/io/detail/read_thread.hpp>
57 #include <osmium/io/detail/read_write.hpp>
58 #include <osmium/io/detail/queue_util.hpp>
59 #include <osmium/io/error.hpp>
60 #include <osmium/io/file.hpp>
61 #include <osmium/io/header.hpp>
62 #include <osmium/memory/buffer.hpp>
64 #include <osmium/thread/util.hpp>
65 #include <osmium/util/config.hpp>
66 
67 namespace osmium {
68 
69  namespace io {
70 
71  namespace detail {
72 
73  inline size_t get_input_queue_size() noexcept {
74  const size_t n = osmium::config::get_max_queue_size("INPUT", 20);
75  return n > 2 ? n : 2;
76  }
77 
78  inline size_t get_osmdata_queue_size() noexcept {
79  const size_t n = osmium::config::get_max_queue_size("OSMDATA", 20);
80  return n > 2 ? n : 2;
81  }
82 
83  } // namespace detail
84 
91  class Reader {
92 
94 
95  detail::ParserFactory::create_parser_type m_creator;
96 
97  enum class status {
98  okay = 0, // normal reading
99  error = 1, // some error occurred while reading
100  closed = 2, // close() called successfully after eof
101  eof = 3 // eof of file was reached without error
102  } m_status;
103 
105 
106  detail::future_string_queue_type m_input_queue;
107 
108  std::unique_ptr<osmium::io::Decompressor> m_decompressor;
109 
110  osmium::io::detail::ReadThreadManager m_read_thread_manager;
111 
112  detail::future_buffer_queue_type m_osmdata_queue;
113  detail::queue_wrapper<osmium::memory::Buffer> m_osmdata_queue_wrapper;
114 
115  std::future<osmium::io::Header> m_header_future;
117 
119 
120  size_t m_file_size;
121 
122  osmium::io::detail::reader_options m_options;
123 
125  m_options.read_which_entities = value;
126  }
127 
128  void set_option(osmium::io::read_meta value) noexcept {
129  m_options.read_metadata = value;
130  }
131 
132  // This function will run in a separate thread.
133  static void parser_thread(const detail::ParserFactory::create_parser_type& creator,
134  detail::future_string_queue_type& input_queue,
135  detail::future_buffer_queue_type& osmdata_queue,
136  std::promise<osmium::io::Header>&& header_promise,
137  osmium::io::detail::reader_options options) {
138  std::promise<osmium::io::Header> promise = std::move(header_promise);
139  const auto parser = creator(input_queue, osmdata_queue, promise, options);
140  parser->parse();
141  }
142 
143 #ifndef _WIN32
144 
155  static int execute(const std::string& command, const std::string& filename, int* childpid) {
156  int pipefd[2];
157  if (pipe(pipefd) < 0) {
158  throw std::system_error(errno, std::system_category(), "opening pipe failed");
159  }
160  const pid_t pid = fork();
161  if (pid < 0) {
162  throw std::system_error(errno, std::system_category(), "fork failed");
163  }
164  if (pid == 0) { // child
165  // close all file descriptors except one end of the pipe
166  for (int i = 0; i < 32; ++i) {
167  if (i != pipefd[1]) {
168  ::close(i);
169  }
170  }
171  if (dup2(pipefd[1], 1) < 0) { // put end of pipe as stdout/stdin
172  exit(1);
173  }
174 
175  ::open("/dev/null", O_RDONLY); // stdin
176  ::open("/dev/null", O_WRONLY); // stderr
177  // hack: -g switches off globbing in curl which allows [] to be used in file names
178  // this is important for XAPI URLs
179  // in theory this execute() function could be used for other commands, but it is
180  // only used for curl at the moment, so this is okay.
181  if (::execlp(command.c_str(), command.c_str(), "-g", filename.c_str(), nullptr) < 0) {
182  exit(1);
183  }
184  }
185  // parent
186  *childpid = pid;
187  ::close(pipefd[1]);
188  return pipefd[0];
189  }
190 #endif
191 
200  static int open_input_file_or_url(const std::string& filename, int* childpid) {
201  std::string protocol = filename.substr(0, filename.find_first_of(':'));
202  if (protocol == "http" || protocol == "https" || protocol == "ftp" || protocol == "file") {
203 #ifndef _WIN32
204  return execute("curl", filename, childpid);
205 #else
206  throw io_error("Reading OSM files from the network currently not supported on Windows.");
207 #endif
208  } else {
209  return osmium::io::detail::open_for_reading(filename);
210  }
211  }
212 
213  public:
214 
237  template <typename... TArgs>
238  explicit Reader(const osmium::io::File& file, TArgs&&... args) :
239  m_file(file.check()),
240  m_creator(detail::ParserFactory::instance().get_creator_function(m_file)),
241  m_status(status::okay),
242  m_childpid(0),
243  m_input_queue(detail::get_input_queue_size(), "raw_input"),
244  m_decompressor(m_file.buffer() ?
245  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
246  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
247  m_read_thread_manager(*m_decompressor, m_input_queue),
248  m_osmdata_queue(detail::get_osmdata_queue_size(), "parser_results"),
249  m_osmdata_queue_wrapper(m_osmdata_queue),
250  m_header_future(),
251  m_header(),
252  m_thread(),
253  m_file_size(m_decompressor->file_size()) {
254 
255  (void)std::initializer_list<int>{
256  (set_option(args), 0)...
257  };
258 
259  std::promise<osmium::io::Header> header_promise;
260  m_header_future = header_promise.get_future();
261  m_thread = osmium::thread::thread_handler{parser_thread, std::ref(m_creator), std::ref(m_input_queue), std::ref(m_osmdata_queue), std::move(header_promise), m_options};
262  }
263 
264  template <typename... TArgs>
265  explicit Reader(const std::string& filename, TArgs&&... args) :
266  Reader(osmium::io::File(filename), std::forward<TArgs>(args)...) {
267  }
268 
269  template <typename... TArgs>
270  explicit Reader(const char* filename, TArgs&&... args) :
271  Reader(osmium::io::File(filename), std::forward<TArgs>(args)...) {
272  }
273 
274  Reader(const Reader&) = delete;
275  Reader& operator=(const Reader&) = delete;
276 
277  Reader(Reader&&) = default;
278  Reader& operator=(Reader&&) = default;
279 
280  ~Reader() noexcept {
281  try {
282  close();
283  } catch (...) {
284  // Ignore any exceptions because destructor must not throw.
285  }
286  }
287 
296  void close() {
298 
299  m_read_thread_manager.stop();
300 
301  m_osmdata_queue_wrapper.drain();
302 
303  try {
304  m_read_thread_manager.close();
305  } catch (...) {
306  // Ignore any exceptions.
307  }
308 
309 #ifndef _WIN32
310  if (m_childpid) {
311  int status;
312  const pid_t pid = ::waitpid(m_childpid, &status, 0);
313 #pragma GCC diagnostic push
314 #pragma GCC diagnostic ignored "-Wold-style-cast"
315  if (pid < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
316  throw std::system_error(errno, std::system_category(), "subprocess returned error");
317  }
318 #pragma GCC diagnostic pop
319  m_childpid = 0;
320  }
321 #endif
322  }
323 
331  if (m_status == status::error) {
332  throw io_error("Can not get header from reader when in status 'error'");
333  }
334 
335  try {
336  if (m_header_future.valid()) {
337  m_header = m_header_future.get();
338  if (m_options.read_which_entities == osmium::osm_entity_bits::nothing) {
340  }
341  }
342  } catch (...) {
343  close();
345  throw;
346  }
347  return m_header;
348  }
349 
361  osmium::memory::Buffer buffer;
362 
363  if (m_status != status::okay ||
364  m_options.read_which_entities == osmium::osm_entity_bits::nothing) {
365  throw io_error("Can not read from reader when in status 'closed', 'eof', or 'error'");
366  }
367 
368  try {
369  // m_input_format.read() can return an invalid buffer to signal EOF,
370  // or a valid buffer with or without data. A valid buffer
371  // without data is not an error, it just means we have to
372  // keep getting the next buffer until there is one with data.
373  while (true) {
374  buffer = m_osmdata_queue_wrapper.pop();
375  if (detail::at_end_of_data(buffer)) {
377  m_read_thread_manager.close();
378  return buffer;
379  }
380  if (buffer.committed() > 0) {
381  return buffer;
382  }
383  }
384  } catch (...) {
385  close();
387  throw;
388  }
389  }
390 
395  bool eof() const {
397  }
398 
403  size_t file_size() const noexcept {
404  return m_file_size;
405  }
406 
421  size_t offset() const noexcept {
422  return m_decompressor->offset();
423  }
424 
425  }; // class Reader
426 
435  template <typename... TArgs>
438 
439  Reader reader(std::forward<TArgs>(args)...);
440  while (osmium::memory::Buffer read_buffer = reader.read()) {
441  buffer.add_buffer(read_buffer);
442  buffer.commit();
443  }
444 
445  return buffer;
446  }
447 
448  } // namespace io
449 
450 } // namespace osmium
451 
452 #endif // OSMIUM_IO_READER_HPP
detail::queue_wrapper< osmium::memory::Buffer > m_osmdata_queue_wrapper
Definition: reader.hpp:113
static void parser_thread(const detail::ParserFactory::create_parser_type &creator, detail::future_string_queue_type &input_queue, detail::future_buffer_queue_type &osmdata_queue, std::promise< osmium::io::Header > &&header_promise, osmium::io::detail::reader_options options)
Definition: reader.hpp:133
status
Definition: reader.hpp:97
osmium::memory::Buffer read()
Definition: reader.hpp:360
type
Definition: entity_bits.hpp:63
int m_childpid
Definition: reader.hpp:104
std::future< osmium::io::Header > m_header_future
Definition: reader.hpp:115
osmium::memory::Buffer read_file(TArgs &&...args)
Definition: reader.hpp:436
Definition: reader_iterator.hpp:39
enum osmium::io::Reader::status m_status
std::unique_ptr< osmium::io::Decompressor > m_decompressor
Definition: reader.hpp:108
Reader & operator=(const Reader &)=delete
bool eof() const
Definition: reader.hpp:395
detail::future_string_queue_type m_input_queue
Definition: reader.hpp:106
osmium::io::File m_file
Definition: reader.hpp:93
Definition: file.hpp:72
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
void set_option(osmium::osm_entity_bits::type value) noexcept
Definition: reader.hpp:124
Definition: attr.hpp:333
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:500
osmium::io::detail::ReadThreadManager m_read_thread_manager
Definition: reader.hpp:110
size_t file_size() const noexcept
Definition: reader.hpp:403
static int execute(const std::string &command, const std::string &filename, int *childpid)
Definition: reader.hpp:155
Definition: reader.hpp:91
~Reader() noexcept
Definition: reader.hpp:280
Definition: error.hpp:44
osmium::io::Header header()
Definition: reader.hpp:330
void set_option(osmium::io::read_meta value) noexcept
Definition: reader.hpp:128
size_t committed() const noexcept
Definition: buffer.hpp:259
size_t get_max_queue_size(const char *queue_name, size_t default_value) noexcept
Definition: config.hpp:69
Definition: buffer.hpp:97
size_t m_file_size
Definition: reader.hpp:120
osmium::thread::thread_handler m_thread
Definition: reader.hpp:118
static int open_input_file_or_url(const std::string &filename, int *childpid)
Definition: reader.hpp:200
detail::future_buffer_queue_type m_osmdata_queue
Definition: reader.hpp:112
osmium::io::detail::reader_options m_options
Definition: reader.hpp:122
osmium::io::Header m_header
Definition: reader.hpp:116
read_meta
Definition: file_format.hpp:52
Definition: entity_bits.hpp:67
Reader(const char *filename, TArgs &&...args)
Definition: reader.hpp:270
Definition: compression.hpp:138
size_t offset() const noexcept
Definition: reader.hpp:421
Reader(const osmium::io::File &file, TArgs &&...args)
Definition: reader.hpp:238
std::string get(const std::string &key, const std::string &default_value="") const noexcept
Definition: options.hpp:125
detail::ParserFactory::create_parser_type m_creator
Definition: reader.hpp:95
Definition: header.hpp:68
Reader(const std::string &filename, TArgs &&...args)
Definition: reader.hpp:265
void close()
Definition: reader.hpp:296
size_t commit()
Definition: buffer.hpp:354
Definition: util.hpp:85