1 | // (C) Copyright Gennadiy Rozental 2004-2005.
|
---|
2 | // Distributed under the Boost Software License, Version 1.0.
|
---|
3 | // (See accompanying file LICENSE_1_0.txt or copy at
|
---|
4 | // http://www.boost.org/LICENSE_1_0.txt)
|
---|
5 |
|
---|
6 | // See http://www.boost.org/libs/test for the library home page.
|
---|
7 | //
|
---|
8 | // File : $RCSfile: token_iterator.hpp,v $
|
---|
9 | //
|
---|
10 | // Version : $Revision: 1.8 $
|
---|
11 | //
|
---|
12 | // Description : token iterator for string and range tokenization
|
---|
13 | // ***************************************************************************
|
---|
14 |
|
---|
15 | #ifndef BOOST_TOKEN_ITERATOR_HPP_071894GER
|
---|
16 | #define BOOST_TOKEN_ITERATOR_HPP_071894GER
|
---|
17 |
|
---|
18 | // Boost
|
---|
19 | #include <boost/config.hpp>
|
---|
20 | #include <boost/detail/workaround.hpp>
|
---|
21 |
|
---|
22 | #include <boost/iterator/iterator_categories.hpp>
|
---|
23 | #include <boost/iterator/iterator_traits.hpp>
|
---|
24 |
|
---|
25 | #include <boost/test/utils/iterator/input_iterator_facade.hpp>
|
---|
26 | #include <boost/test/utils/basic_cstring/basic_cstring.hpp>
|
---|
27 | #include <boost/test/utils/named_params.hpp>
|
---|
28 | #include <boost/test/utils/foreach.hpp>
|
---|
29 |
|
---|
30 | // STL
|
---|
31 | #include <iosfwd>
|
---|
32 | #include <cctype>
|
---|
33 |
|
---|
34 | #include <boost/test/detail/suppress_warnings.hpp>
|
---|
35 |
|
---|
36 | //____________________________________________________________________________//
|
---|
37 |
|
---|
38 | #ifdef BOOST_NO_STDC_NAMESPACE
|
---|
39 | namespace std{ using ::ispunct; using ::isspace; }
|
---|
40 | #endif
|
---|
41 |
|
---|
42 | namespace boost {
|
---|
43 |
|
---|
44 | namespace unit_test {
|
---|
45 |
|
---|
46 | // ************************************************************************** //
|
---|
47 | // ************** ti_delimeter_type ************** //
|
---|
48 | // ************************************************************************** //
|
---|
49 |
|
---|
50 | enum ti_delimeter_type {
|
---|
51 | dt_char, // character is delimeter if it among explicit list of some characters
|
---|
52 | dt_ispunct, // character is delimeter if it satisfies ispunct functor
|
---|
53 | dt_isspace, // character is delimeter if it satisfies isspace functor
|
---|
54 | dt_none // no character is delimeter
|
---|
55 | };
|
---|
56 |
|
---|
57 | namespace ut_detail {
|
---|
58 |
|
---|
59 | // ************************************************************************** //
|
---|
60 | // ************** default_char_compare ************** //
|
---|
61 | // ************************************************************************** //
|
---|
62 |
|
---|
63 | template<typename CharT>
|
---|
64 | class default_char_compare {
|
---|
65 | public:
|
---|
66 | bool operator()( CharT c1, CharT c2 )
|
---|
67 | {
|
---|
68 | #ifdef BOOST_CLASSIC_IOSTREAMS
|
---|
69 | return std::string_char_traits<CharT>::eq( c1, c2 );
|
---|
70 | #else
|
---|
71 | return std::char_traits<CharT>::eq( c1, c2 );
|
---|
72 | #endif
|
---|
73 | }
|
---|
74 | };
|
---|
75 |
|
---|
76 | // ************************************************************************** //
|
---|
77 | // ************** delim_policy ************** //
|
---|
78 | // ************************************************************************** //
|
---|
79 |
|
---|
80 | template<typename CharT,typename CharCompare>
|
---|
81 | class delim_policy {
|
---|
82 | typedef basic_cstring<CharT const> cstring;
|
---|
83 | public:
|
---|
84 | // Constructor
|
---|
85 | explicit delim_policy( ti_delimeter_type t = dt_char, cstring d = cstring() )
|
---|
86 | : m_type( t )
|
---|
87 | {
|
---|
88 | set_delimeters( d );
|
---|
89 | }
|
---|
90 |
|
---|
91 | void set_delimeters( ti_delimeter_type t ) { m_type = t; }
|
---|
92 | template<typename Src>
|
---|
93 | void set_delimeters( Src d )
|
---|
94 | {
|
---|
95 | nfp::optionally_assign( m_delimeters, d );
|
---|
96 |
|
---|
97 | if( !m_delimeters.is_empty() )
|
---|
98 | m_type = dt_char;
|
---|
99 | }
|
---|
100 |
|
---|
101 | bool operator()( CharT c )
|
---|
102 | {
|
---|
103 | switch( m_type ) {
|
---|
104 | case dt_char: {
|
---|
105 | BOOST_TEST_FOREACH( CharT, delim, m_delimeters )
|
---|
106 | if( CharCompare()( delim, c ) )
|
---|
107 | return true;
|
---|
108 |
|
---|
109 | return false;
|
---|
110 | }
|
---|
111 | case dt_ispunct:
|
---|
112 | return (std::ispunct)( c ) != 0;
|
---|
113 | case dt_isspace:
|
---|
114 | return (std::isspace)( c ) != 0;
|
---|
115 | case dt_none:
|
---|
116 | return false;
|
---|
117 | }
|
---|
118 |
|
---|
119 | return false;
|
---|
120 | }
|
---|
121 |
|
---|
122 | private:
|
---|
123 | // Data members
|
---|
124 | cstring m_delimeters;
|
---|
125 | ti_delimeter_type m_type;
|
---|
126 | };
|
---|
127 |
|
---|
128 | // ************************************************************************** //
|
---|
129 | // ************** token_assigner ************** //
|
---|
130 | // ************************************************************************** //
|
---|
131 |
|
---|
132 | template<typename TraversalTag>
|
---|
133 | struct token_assigner {
|
---|
134 | #if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 )
|
---|
135 | template<typename Iterator, typename C, typename T>
|
---|
136 | static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t )
|
---|
137 | { for( ; b != e; ++b ) t += *b; }
|
---|
138 |
|
---|
139 | template<typename Iterator, typename C>
|
---|
140 | static void assign( Iterator b, Iterator e, basic_cstring<C>& t ) { t.assign( b, e ); }
|
---|
141 | #else
|
---|
142 | template<typename Iterator, typename Token>
|
---|
143 | static void assign( Iterator b, Iterator e, Token& t ) { t.assign( b, e ); }
|
---|
144 | #endif
|
---|
145 | template<typename Iterator, typename Token>
|
---|
146 | static void append_move( Iterator& b, Token& ) { ++b; }
|
---|
147 | };
|
---|
148 |
|
---|
149 | //____________________________________________________________________________//
|
---|
150 |
|
---|
151 | template<>
|
---|
152 | struct token_assigner<single_pass_traversal_tag> {
|
---|
153 | template<typename Iterator, typename Token>
|
---|
154 | static void assign( Iterator b, Iterator e, Token& t ) {}
|
---|
155 |
|
---|
156 | template<typename Iterator, typename Token>
|
---|
157 | static void append_move( Iterator& b, Token& t ) { t += *b; ++b; }
|
---|
158 | };
|
---|
159 |
|
---|
160 | } // namespace ut_detail
|
---|
161 |
|
---|
162 | // ************************************************************************** //
|
---|
163 | // ************** modifiers ************** //
|
---|
164 | // ************************************************************************** //
|
---|
165 |
|
---|
166 | namespace {
|
---|
167 | nfp::keyword<struct dropped_delimeters_t > dropped_delimeters;
|
---|
168 | nfp::keyword<struct kept_delimeters_t > kept_delimeters;
|
---|
169 | nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens;
|
---|
170 | nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens;
|
---|
171 | }
|
---|
172 |
|
---|
173 | // ************************************************************************** //
|
---|
174 | // ************** token_iterator_base ************** //
|
---|
175 | // ************************************************************************** //
|
---|
176 |
|
---|
177 | template<typename Derived,
|
---|
178 | typename CharT,
|
---|
179 | typename CharCompare = ut_detail::default_char_compare<CharT>,
|
---|
180 | typename ValueType = basic_cstring<CharT const>,
|
---|
181 | typename Reference = basic_cstring<CharT const>,
|
---|
182 | typename Traversal = forward_traversal_tag>
|
---|
183 | class token_iterator_base
|
---|
184 | : public input_iterator_facade<Derived,ValueType,Reference,Traversal> {
|
---|
185 | typedef basic_cstring<CharT const> cstring;
|
---|
186 | typedef ut_detail::delim_policy<CharT,CharCompare> delim_policy;
|
---|
187 | typedef input_iterator_facade<Derived,ValueType,Reference,Traversal> base;
|
---|
188 |
|
---|
189 | protected:
|
---|
190 | // Constructor
|
---|
191 | explicit token_iterator_base()
|
---|
192 | : m_is_dropped( dt_isspace )
|
---|
193 | , m_is_kept( dt_ispunct )
|
---|
194 | , m_keep_empty_tokens( false )
|
---|
195 | , m_tokens_left( (std::size_t)-1 )
|
---|
196 | , m_token_produced( false )
|
---|
197 | {
|
---|
198 | }
|
---|
199 |
|
---|
200 | template<typename Modifier>
|
---|
201 | void
|
---|
202 | apply_modifier( Modifier const& m )
|
---|
203 | {
|
---|
204 | if( m.has( dropped_delimeters ) )
|
---|
205 | m_is_dropped.set_delimeters( m[dropped_delimeters] );
|
---|
206 |
|
---|
207 | if( m.has( kept_delimeters ) )
|
---|
208 | m_is_kept.set_delimeters( m[kept_delimeters] );
|
---|
209 |
|
---|
210 | if( m.has( keep_empty_tokens ) )
|
---|
211 | m_keep_empty_tokens = true;
|
---|
212 |
|
---|
213 | nfp::optionally_assign( m_tokens_left, m, max_tokens );
|
---|
214 | }
|
---|
215 |
|
---|
216 | template<typename Iter>
|
---|
217 | bool get( Iter& begin, Iter end )
|
---|
218 | {
|
---|
219 | typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner;
|
---|
220 | Iter checkpoint;
|
---|
221 |
|
---|
222 | this->m_value.clear();
|
---|
223 |
|
---|
224 | if( !m_keep_empty_tokens ) {
|
---|
225 | while( begin != end && m_is_dropped( *begin ) )
|
---|
226 | ++begin;
|
---|
227 |
|
---|
228 | if( begin == end )
|
---|
229 | return false;
|
---|
230 |
|
---|
231 | checkpoint = begin;
|
---|
232 |
|
---|
233 | if( m_tokens_left == 1 )
|
---|
234 | while( begin != end )
|
---|
235 | Assigner::append_move( begin, this->m_value );
|
---|
236 | else if( m_is_kept( *begin ) )
|
---|
237 | Assigner::append_move( begin, this->m_value );
|
---|
238 | else
|
---|
239 | while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
|
---|
240 | Assigner::append_move( begin, this->m_value );
|
---|
241 |
|
---|
242 | --m_tokens_left;
|
---|
243 | }
|
---|
244 | else { // m_keep_empty_tokens is true
|
---|
245 | checkpoint = begin;
|
---|
246 |
|
---|
247 | if( begin == end ) {
|
---|
248 | if( m_token_produced )
|
---|
249 | return false;
|
---|
250 |
|
---|
251 | m_token_produced = true;
|
---|
252 | }
|
---|
253 | if( m_is_kept( *begin ) ) {
|
---|
254 | if( m_token_produced )
|
---|
255 | Assigner::append_move( begin, this->m_value );
|
---|
256 |
|
---|
257 | m_token_produced = !m_token_produced;
|
---|
258 | }
|
---|
259 | else if( !m_token_produced && m_is_dropped( *begin ) )
|
---|
260 | m_token_produced = true;
|
---|
261 | else {
|
---|
262 | if( m_is_dropped( *begin ) )
|
---|
263 | checkpoint = ++begin;
|
---|
264 |
|
---|
265 | while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
|
---|
266 | Assigner::append_move( begin, this->m_value );
|
---|
267 |
|
---|
268 | m_token_produced = true;
|
---|
269 | }
|
---|
270 | }
|
---|
271 |
|
---|
272 | Assigner::assign( checkpoint, begin, this->m_value );
|
---|
273 |
|
---|
274 | return true;
|
---|
275 | }
|
---|
276 |
|
---|
277 | private:
|
---|
278 | // Data members
|
---|
279 | delim_policy m_is_dropped;
|
---|
280 | delim_policy m_is_kept;
|
---|
281 | bool m_keep_empty_tokens;
|
---|
282 | std::size_t m_tokens_left;
|
---|
283 | bool m_token_produced;
|
---|
284 | };
|
---|
285 |
|
---|
286 | // ************************************************************************** //
|
---|
287 | // ************** basic_string_token_iterator ************** //
|
---|
288 | // ************************************************************************** //
|
---|
289 |
|
---|
290 | template<typename CharT,
|
---|
291 | typename CharCompare = ut_detail::default_char_compare<CharT> >
|
---|
292 | class basic_string_token_iterator
|
---|
293 | : public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> {
|
---|
294 | typedef basic_cstring<CharT const> cstring;
|
---|
295 | typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base;
|
---|
296 | public:
|
---|
297 | explicit basic_string_token_iterator() {}
|
---|
298 | explicit basic_string_token_iterator( cstring src )
|
---|
299 | : m_src( src )
|
---|
300 | {
|
---|
301 | this->init();
|
---|
302 | }
|
---|
303 |
|
---|
304 | template<typename Src, typename Modifier>
|
---|
305 | basic_string_token_iterator( Src src, Modifier const& m )
|
---|
306 | : m_src( src )
|
---|
307 | {
|
---|
308 | this->apply_modifier( m );
|
---|
309 |
|
---|
310 | this->init();
|
---|
311 | }
|
---|
312 |
|
---|
313 | private:
|
---|
314 | friend class input_iterator_core_access;
|
---|
315 |
|
---|
316 | // input iterator implementation
|
---|
317 | bool get()
|
---|
318 | {
|
---|
319 | typename cstring::iterator begin = m_src.begin();
|
---|
320 | bool res = base::get( begin, m_src.end() );
|
---|
321 |
|
---|
322 | m_src.assign( begin, m_src.end() );
|
---|
323 |
|
---|
324 | return res;
|
---|
325 | }
|
---|
326 |
|
---|
327 | // Data members
|
---|
328 | cstring m_src;
|
---|
329 | };
|
---|
330 |
|
---|
331 | typedef basic_string_token_iterator<char> string_token_iterator;
|
---|
332 | typedef basic_string_token_iterator<wchar_t> wstring_token_iterator;
|
---|
333 |
|
---|
334 | // ************************************************************************** //
|
---|
335 | // ************** range_token_iterator ************** //
|
---|
336 | // ************************************************************************** //
|
---|
337 |
|
---|
338 | template<typename Iter,
|
---|
339 | typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
|
---|
340 | typename ValueType = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
|
---|
341 | typename Reference = ValueType const&>
|
---|
342 | class range_token_iterator
|
---|
343 | : public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
|
---|
344 | typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> {
|
---|
345 | typedef basic_cstring<typename ValueType::value_type> cstring;
|
---|
346 | typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
|
---|
347 | typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base;
|
---|
348 | public:
|
---|
349 | explicit range_token_iterator() {}
|
---|
350 | explicit range_token_iterator( Iter begin, Iter end = Iter() )
|
---|
351 | : m_begin( begin ), m_end( end )
|
---|
352 | {
|
---|
353 | this->init();
|
---|
354 | }
|
---|
355 | range_token_iterator( range_token_iterator const& rhs )
|
---|
356 | : base( rhs )
|
---|
357 | {
|
---|
358 | if( this->m_valid ) {
|
---|
359 | m_begin = rhs.m_begin;
|
---|
360 | m_end = rhs.m_end;
|
---|
361 | }
|
---|
362 | }
|
---|
363 |
|
---|
364 | template<typename Modifier>
|
---|
365 | range_token_iterator( Iter begin, Iter end, Modifier const& m )
|
---|
366 | : m_begin( begin ), m_end( end )
|
---|
367 | {
|
---|
368 | this->apply_modifier( m );
|
---|
369 |
|
---|
370 | this->init();
|
---|
371 | }
|
---|
372 |
|
---|
373 | private:
|
---|
374 | friend class input_iterator_core_access;
|
---|
375 |
|
---|
376 | // input iterator implementation
|
---|
377 | bool get()
|
---|
378 | {
|
---|
379 | return base::get( m_begin, m_end );
|
---|
380 | }
|
---|
381 |
|
---|
382 | // Data members
|
---|
383 | Iter m_begin;
|
---|
384 | Iter m_end;
|
---|
385 | };
|
---|
386 |
|
---|
387 | // ************************************************************************** //
|
---|
388 | // ************** make_range_token_iterator ************** //
|
---|
389 | // ************************************************************************** //
|
---|
390 |
|
---|
391 | template<typename Iter>
|
---|
392 | inline range_token_iterator<Iter>
|
---|
393 | make_range_token_iterator( Iter begin, Iter end = Iter() )
|
---|
394 | {
|
---|
395 | return range_token_iterator<Iter>( begin, end );
|
---|
396 | }
|
---|
397 |
|
---|
398 | //____________________________________________________________________________//
|
---|
399 |
|
---|
400 | template<typename Iter,typename Modifier>
|
---|
401 | inline range_token_iterator<Iter>
|
---|
402 | make_range_token_iterator( Iter begin, Iter end, Modifier const& m )
|
---|
403 | {
|
---|
404 | return range_token_iterator<Iter>( begin, end, m );
|
---|
405 | }
|
---|
406 |
|
---|
407 | //____________________________________________________________________________//
|
---|
408 |
|
---|
409 | } // namespace unit_test
|
---|
410 |
|
---|
411 | } // namespace boost
|
---|
412 |
|
---|
413 | //____________________________________________________________________________//
|
---|
414 |
|
---|
415 | #include <boost/test/detail/enable_warnings.hpp>
|
---|
416 |
|
---|
417 | // ***************************************************************************
|
---|
418 | // Revision History :
|
---|
419 | //
|
---|
420 | // $Log: token_iterator.hpp,v $
|
---|
421 | // Revision 1.8 2005/06/16 05:58:26 rogeeff
|
---|
422 | // make default constructed range token iterator copyable according ot standard
|
---|
423 | //
|
---|
424 | // Revision 1.7 2005/06/11 19:23:28 rogeeff
|
---|
425 | // 1. Always use clear
|
---|
426 | // reorder field in constructor to eliminate warning
|
---|
427 | // remove cw workaround - doesn't seems to be needed
|
---|
428 | //
|
---|
429 | // Revision 1.6 2005/06/05 16:07:51 grafik
|
---|
430 | // Work around CW-8 problem parsing the switch statement.
|
---|
431 | //
|
---|
432 | // Revision 1.5 2005/04/12 06:46:42 rogeeff
|
---|
433 | // use named_param facilites
|
---|
434 | //
|
---|
435 | // Revision 1.4 2005/02/20 08:27:09 rogeeff
|
---|
436 | // This a major update for Boost.Test framework. See release docs for complete list of fixes/updates
|
---|
437 | //
|
---|
438 | // Revision 1.3 2005/02/01 06:40:08 rogeeff
|
---|
439 | // copyright update
|
---|
440 | // old log entries removed
|
---|
441 | // minor stylitic changes
|
---|
442 | // deprecated tools removed
|
---|
443 | //
|
---|
444 | // Revision 1.2 2005/01/22 19:22:14 rogeeff
|
---|
445 | // implementation moved into headers section to eliminate dependency of included/minimal component on src directory
|
---|
446 | //
|
---|
447 | // Revision 1.1 2005/01/22 18:21:40 rogeeff
|
---|
448 | // moved sharable staff into utils
|
---|
449 | //
|
---|
450 | // ***************************************************************************
|
---|
451 |
|
---|
452 | #endif // BOOST_TOKEN_ITERATOR_HPP_071894GER
|
---|
453 |
|
---|