source: NonGTP/Boost/boost/test/utils/iterator/token_iterator.hpp @ 857

Revision 857, 15.2 KB checked in by igarcia, 18 years ago (diff)
Line 
1//  (C) Copyright Gennadiy Rozental 2004-2005.
2//  Distributed under the Boost Software License, Version 1.0.
3//  (See accompanying file LICENSE_1_0.txt or copy at
4//  http://www.boost.org/LICENSE_1_0.txt)
5
6//  See http://www.boost.org/libs/test for the library home page.
7//
8//  File        : $RCSfile: token_iterator.hpp,v $
9//
10//  Version     : $Revision: 1.8 $
11//
12//  Description : token iterator for string and range tokenization
13// ***************************************************************************
14
15#ifndef BOOST_TOKEN_ITERATOR_HPP_071894GER
16#define BOOST_TOKEN_ITERATOR_HPP_071894GER
17
18// Boost
19#include <boost/config.hpp>
20#include <boost/detail/workaround.hpp>
21
22#include <boost/iterator/iterator_categories.hpp>
23#include <boost/iterator/iterator_traits.hpp>
24
25#include <boost/test/utils/iterator/input_iterator_facade.hpp>
26#include <boost/test/utils/basic_cstring/basic_cstring.hpp>
27#include <boost/test/utils/named_params.hpp>
28#include <boost/test/utils/foreach.hpp>
29
30// STL
31#include <iosfwd>
32#include <cctype>
33
34#include <boost/test/detail/suppress_warnings.hpp>
35
36//____________________________________________________________________________//
37
38#ifdef BOOST_NO_STDC_NAMESPACE
39namespace std{ using ::ispunct; using ::isspace; }
40#endif
41
42namespace boost {
43
44namespace unit_test {
45
46// ************************************************************************** //
47// **************               ti_delimeter_type              ************** //
48// ************************************************************************** //
49
50enum ti_delimeter_type {
51    dt_char,        // character is delimeter if it among explicit list of some characters
52    dt_ispunct,     // character is delimeter if it satisfies ispunct functor
53    dt_isspace,     // character is delimeter if it satisfies isspace functor
54    dt_none         // no character is delimeter
55};
56
57namespace ut_detail {
58
59// ************************************************************************** //
60// **************             default_char_compare             ************** //
61// ************************************************************************** //
62
63template<typename CharT>
64class default_char_compare {
65public:
66    bool operator()( CharT c1, CharT c2 )
67    {
68#ifdef BOOST_CLASSIC_IOSTREAMS
69        return std::string_char_traits<CharT>::eq( c1, c2 );
70#else
71        return std::char_traits<CharT>::eq( c1, c2 );
72#endif
73    }
74};
75
76// ************************************************************************** //
77// **************                 delim_policy                 ************** //
78// ************************************************************************** //
79
80template<typename CharT,typename CharCompare>
81class delim_policy {
82    typedef basic_cstring<CharT const> cstring;
83public:
84    // Constructor
85    explicit    delim_policy( ti_delimeter_type t = dt_char, cstring d = cstring() )
86    : m_type( t )
87    {
88        set_delimeters( d );
89    }
90
91    void        set_delimeters( ti_delimeter_type t ) { m_type = t; }
92    template<typename Src>
93    void        set_delimeters( Src d )
94    {
95        nfp::optionally_assign( m_delimeters, d );
96       
97        if( !m_delimeters.is_empty() )
98            m_type = dt_char;
99    }
100
101    bool        operator()( CharT c )
102    {
103        switch( m_type ) {
104        case dt_char: {
105            BOOST_TEST_FOREACH( CharT, delim, m_delimeters )
106                if( CharCompare()( delim, c ) )
107                    return true;
108
109            return false;
110        }
111        case dt_ispunct:
112            return (std::ispunct)( c ) != 0;
113        case dt_isspace:
114            return (std::isspace)( c ) != 0;
115        case dt_none:
116            return false;
117        }
118
119        return false;
120    }
121
122private:
123    // Data members
124    cstring             m_delimeters;
125    ti_delimeter_type   m_type;
126};
127
128// ************************************************************************** //
129// **************                 token_assigner               ************** //
130// ************************************************************************** //
131
132template<typename TraversalTag>
133struct token_assigner {
134#if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 )
135    template<typename Iterator, typename C, typename T>
136    static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t )
137    { for( ; b != e; ++b ) t += *b; }
138   
139    template<typename Iterator, typename C>
140    static void assign( Iterator b, Iterator e, basic_cstring<C>& t )  { t.assign( b, e ); }
141#else
142    template<typename Iterator, typename Token>
143    static void assign( Iterator b, Iterator e, Token& t )  { t.assign( b, e ); }
144#endif
145    template<typename Iterator, typename Token>
146    static void append_move( Iterator& b, Token& )          { ++b; }
147};
148
149//____________________________________________________________________________//
150
151template<>
152struct token_assigner<single_pass_traversal_tag> {
153    template<typename Iterator, typename Token>
154    static void assign( Iterator b, Iterator e, Token& t )  {}
155
156    template<typename Iterator, typename Token>
157    static void append_move( Iterator& b, Token& t )        { t += *b; ++b; }
158};
159
160} // namespace ut_detail
161
162// ************************************************************************** //
163// **************                  modifiers                   ************** //
164// ************************************************************************** //
165
166namespace {
167nfp::keyword<struct dropped_delimeters_t > dropped_delimeters;
168nfp::keyword<struct kept_delimeters_t > kept_delimeters;
169nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens;
170nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens;
171}
172
173// ************************************************************************** //
174// **************             token_iterator_base              ************** //
175// ************************************************************************** //
176
177template<typename Derived,
178         typename CharT,
179         typename CharCompare   = ut_detail::default_char_compare<CharT>,
180         typename ValueType     = basic_cstring<CharT const>,
181         typename Reference     = basic_cstring<CharT const>,
182         typename Traversal     = forward_traversal_tag>
183class token_iterator_base
184: public input_iterator_facade<Derived,ValueType,Reference,Traversal> {
185    typedef basic_cstring<CharT const>                                      cstring;
186    typedef ut_detail::delim_policy<CharT,CharCompare>                      delim_policy;
187    typedef input_iterator_facade<Derived,ValueType,Reference,Traversal>    base;
188
189protected:
190    // Constructor
191    explicit    token_iterator_base()
192    : m_is_dropped( dt_isspace )
193    , m_is_kept( dt_ispunct )
194    , m_keep_empty_tokens( false )
195    , m_tokens_left( (std::size_t)-1 )
196    , m_token_produced( false )
197    {
198    }
199
200    template<typename Modifier>
201    void
202    apply_modifier( Modifier const& m )
203    {
204        if( m.has( dropped_delimeters ) )
205            m_is_dropped.set_delimeters( m[dropped_delimeters] );
206
207        if( m.has( kept_delimeters ) )
208            m_is_kept.set_delimeters( m[kept_delimeters] );
209
210        if( m.has( keep_empty_tokens ) )
211            m_keep_empty_tokens = true;
212
213        nfp::optionally_assign( m_tokens_left, m, max_tokens );
214    }
215
216    template<typename Iter>
217    bool                    get( Iter& begin, Iter end )
218    {
219        typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner;
220        Iter checkpoint;
221
222        this->m_value.clear();
223
224        if( !m_keep_empty_tokens ) {
225            while( begin != end && m_is_dropped( *begin ) )
226                ++begin;
227
228            if( begin == end )
229                return false;
230
231            checkpoint = begin;
232
233            if( m_tokens_left == 1 )
234                while( begin != end )
235                    Assigner::append_move( begin, this->m_value );
236            else if( m_is_kept( *begin ) )
237                Assigner::append_move( begin, this->m_value );
238            else
239                while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
240                    Assigner::append_move( begin, this->m_value );
241
242            --m_tokens_left;
243        }
244        else { // m_keep_empty_tokens is true
245            checkpoint = begin;
246
247            if( begin == end ) {
248                if( m_token_produced )
249                    return false;
250
251                m_token_produced = true;
252            }
253            if( m_is_kept( *begin ) ) {
254                if( m_token_produced )
255                    Assigner::append_move( begin, this->m_value );
256
257                m_token_produced = !m_token_produced;
258            }
259            else if( !m_token_produced && m_is_dropped( *begin ) )
260                m_token_produced = true;
261            else {
262                if( m_is_dropped( *begin ) )
263                    checkpoint = ++begin;
264
265                while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
266                    Assigner::append_move( begin, this->m_value );
267
268                m_token_produced = true;
269            }
270        }
271
272        Assigner::assign( checkpoint, begin, this->m_value );
273
274        return true;
275    }
276
277private:
278    // Data members
279    delim_policy            m_is_dropped;
280    delim_policy            m_is_kept;
281    bool                    m_keep_empty_tokens;
282    std::size_t             m_tokens_left;
283    bool                    m_token_produced;
284};
285
286// ************************************************************************** //
287// **************          basic_string_token_iterator         ************** //
288// ************************************************************************** //
289
290template<typename CharT,
291         typename CharCompare = ut_detail::default_char_compare<CharT> >
292class basic_string_token_iterator
293: public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> {
294    typedef basic_cstring<CharT const> cstring;
295    typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base;
296public:
297    explicit    basic_string_token_iterator() {}
298    explicit    basic_string_token_iterator( cstring src )
299    : m_src( src )
300    {
301        this->init();
302    }
303
304    template<typename Src, typename Modifier>
305    basic_string_token_iterator( Src src, Modifier const& m )
306    : m_src( src )
307    {
308        this->apply_modifier( m );
309
310        this->init();
311    }
312
313private:
314    friend class input_iterator_core_access;
315
316    // input iterator implementation
317    bool        get()
318    {
319        typename cstring::iterator begin = m_src.begin();
320        bool res = base::get( begin, m_src.end() );
321
322        m_src.assign( begin, m_src.end() );
323
324        return res;
325    }
326
327    // Data members
328    cstring     m_src;
329};
330
331typedef basic_string_token_iterator<char>       string_token_iterator;
332typedef basic_string_token_iterator<wchar_t>    wstring_token_iterator;
333
334// ************************************************************************** //
335// **************              range_token_iterator            ************** //
336// ************************************************************************** //
337
338template<typename Iter,
339         typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
340         typename ValueType   = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
341         typename Reference   = ValueType const&>
342class range_token_iterator
343: public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
344                             typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> {
345    typedef basic_cstring<typename ValueType::value_type> cstring;
346    typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
347                                typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base;
348public:
349    explicit    range_token_iterator() {}
350    explicit    range_token_iterator( Iter begin, Iter end = Iter() )
351    : m_begin( begin ), m_end( end )
352    {
353        this->init();
354    }
355    range_token_iterator( range_token_iterator const& rhs )
356    : base( rhs )
357    {
358        if( this->m_valid ) {
359            m_begin = rhs.m_begin;
360            m_end   = rhs.m_end;
361        }
362    }
363
364    template<typename Modifier>
365    range_token_iterator( Iter begin, Iter end, Modifier const& m )
366    : m_begin( begin ), m_end( end )
367    {
368        this->apply_modifier( m );
369
370        this->init();
371    }
372
373private:
374    friend class input_iterator_core_access;
375
376    // input iterator implementation
377    bool        get()
378    {
379        return base::get( m_begin, m_end );
380    }
381
382    // Data members
383    Iter m_begin;
384    Iter m_end;
385};
386
387// ************************************************************************** //
388// **************            make_range_token_iterator         ************** //
389// ************************************************************************** //
390
391template<typename Iter>
392inline range_token_iterator<Iter>
393make_range_token_iterator( Iter begin, Iter end = Iter() )
394{
395    return range_token_iterator<Iter>( begin, end );
396}
397
398//____________________________________________________________________________//
399
400template<typename Iter,typename Modifier>
401inline range_token_iterator<Iter>
402make_range_token_iterator( Iter begin, Iter end, Modifier const& m )
403{
404    return range_token_iterator<Iter>( begin, end, m );
405}
406
407//____________________________________________________________________________//
408
409} // namespace unit_test
410
411} // namespace boost
412
413//____________________________________________________________________________//
414
415#include <boost/test/detail/enable_warnings.hpp>
416
417// ***************************************************************************
418//  Revision History :
419// 
420//  $Log: token_iterator.hpp,v $
421//  Revision 1.8  2005/06/16 05:58:26  rogeeff
422//  make default constructed range token iterator copyable according ot standard
423//
424//  Revision 1.7  2005/06/11 19:23:28  rogeeff
425//  1. Always use clear
426//  reorder field in constructor to eliminate warning
427//  remove cw workaround - doesn't seems to be needed
428//
429//  Revision 1.6  2005/06/05 16:07:51  grafik
430//  Work around CW-8 problem parsing the switch statement.
431//
432//  Revision 1.5  2005/04/12 06:46:42  rogeeff
433//  use named_param facilites
434//
435//  Revision 1.4  2005/02/20 08:27:09  rogeeff
436//  This a major update for Boost.Test framework. See release docs for complete list of fixes/updates
437//
438//  Revision 1.3  2005/02/01 06:40:08  rogeeff
439//  copyright update
440//  old log entries removed
441//  minor stylitic changes
442//  deprecated tools removed
443//
444//  Revision 1.2  2005/01/22 19:22:14  rogeeff
445//  implementation moved into headers section to eliminate dependency of included/minimal component on src directory
446//
447//  Revision 1.1  2005/01/22 18:21:40  rogeeff
448//  moved sharable staff into utils
449//
450// ***************************************************************************
451
452#endif // BOOST_TOKEN_ITERATOR_HPP_071894GER
453
Note: See TracBrowser for help on using the repository browser.