1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 """
18 Configuration Handling
19 ======================
20
21 This modules handles configuration loading and provides an easy API
22 for accessing it.
23 """
24 __author__ = u"Andr\xe9 Malo"
25 __docformat__ = "restructuredtext en"
26
27 import os as _os
28 import re as _re
29 import sys as _sys
30
31 from wtf import Error
32
33
35 """ Configuration error """
36
38 """ Config file IO error """
39
41 """
42 Parse error
43
44 :CVariables:
45 - `_MESSAGE`: The message format string
46
47 :IVariables:
48 - `filename`: The name of the file where the error occured
49 - `lineno`: The line number of the error
50
51 :Types:
52 - `_MESSAGE`: ``str``
53 - `filename`: ``basestring``
54 - `lineno`: ``int``
55 """
56 _MESSAGE = "Parse error in %(filename)r, line %(lineno)s"
57
59 """
60 Initialization
61
62 :Parameters:
63 - `filename`: The name of the file, where the error occured
64 - `lineno`: The erroneous line number
65
66 :Types:
67 - `filename`: ``basestring``
68 - `lineno`: ``int``
69 """
70 ConfigurationError.__init__(self, filename, lineno)
71 self.filename = filename
72 self.lineno = lineno
73 self._param = dict(filename=filename, lineno=lineno)
74
76 """ Returns a string representation of the Exception """
77 return self._MESSAGE % self._param
78
80 """ A line continuation without a previous option line occured """
81 _MESSAGE = "Invalid line continuation in %(filename)r, line %(lineno)s"
82
84 """ A option line could not be parsed """
85 _MESSAGE = "Option syntax error in %(filename)r, line %(lineno)s"
86
88 """ Recursive Include Detected """
89 _MESSAGE = "Recursive include detected in %(filename)r, line " \
90 "%(lineno)d: %(included)r"
91
92 - def __init__(self, filename, lineno, included):
93 """
94 Initialization
95
96 :Parameters:
97 - `filename`: The name of the file, where the error occured
98 - `lineno`: The erroneous line number
99 - `included`: recursively included file
100
101 :Types:
102 - `filename`: ``basestring``
103 - `lineno`: ``int``
104 - `included`: ``basestring``
105 """
106 ParseError.__init__(self, filename, lineno)
107 self.included = included
108 self._param['included'] = included
109
111 """ An option type could not be recognized """
112 _MESSAGE = "Failed option type conversion"
113
114
116 """
117 Simplified config file parser
118
119 The ``ConfigParser`` module does too much magic (partially
120 not even documented). Further we don't need all the set and
121 save stuff here, so we write our own - clean - variant.
122 This variant just reads the stuff and does not apply any
123 typing or transformation. It also uses a better design...
124
125 :IVariables:
126 - `_config`: Config instance to feed
127 - `_charset`: Default config charset
128
129 :Types:
130 - `_config`: `Config`
131 - `_charset`: ``str``
132 """
133
134 - def __init__(self, config, charset='latin-1'):
135 """
136 Initialization
137
138 :Parameters:
139 - `config`: Config instance
140 - `charset`: Default config charset
141
142 :Types:
143 - `config`: `Config`
144 - `charset`: ``str``
145 """
146 self._config, self._charset = config, charset
147
148 - def parse(self, fp, filename, _included=None):
149 """
150 Reads from `fp` until EOF and parses line by line
151
152 :Parameters:
153 - `fp`: The stream to read from
154 - `filename`: The filename used for relative includes and
155 error messages
156 - `_included`: Set of already included filenames for recursion check
157
158 :Types:
159 - `fp`: ``file``
160 - `filename`: ``basestring``
161 - `_included`: ``set``
162
163 :Exceptions:
164 - `ContinuationError`: An invalid line continuation occured
165 - `OptionSyntaxError`: An option line could not be parsed
166 - `IOError`: An I/O error occured while reading the stream
167 """
168
169
170 lineno, section, option = 0, None, None
171 root_section, charset, includes = None, self._charset, ()
172
173
174 config = self._config
175 readline = fp.readline
176 is_comment = self._is_comment
177 try_section = self._try_section
178 parse = self._parse_option
179 make_section = self._make_section
180
181 def handle_root(root_section):
182 """ Handle root section """
183 if root_section is None:
184 return charset, includes, None
185 self._cast(root_section)
186 _charset, _includes = charset, []
187 if u'charset' in root_section:
188 _charset = list(root_section.charset)
189 if len(_charset) != 1:
190 raise ContinuationError("Invalid charset declaration")
191 _charset = _charset[0].encode('ascii')
192 if u'include' in root_section:
193 _includes = list(root_section.include)
194 return _charset, _includes, None
195
196 while True:
197 line = readline()
198 if not line:
199 break
200 line = line.decode(charset)
201 lineno += 1
202
203
204 if line.strip() and not is_comment(line):
205
206 header = try_section(line)
207 if header is not None:
208 charset, includes, root_section = \
209 handle_root(root_section)
210 option = None
211 header = header.strip()
212 if header in config:
213 section = config[header]
214 else:
215 config[header] = section = make_section()
216
217
218 elif line[0].isspace():
219 if option is None:
220 raise ContinuationError(filename, lineno)
221 option.append(line.strip())
222
223
224 else:
225 name, value = parse(line)
226 name = name.strip()
227 if not name:
228 raise OptionSyntaxError(filename, lineno)
229 option = [value]
230 if section is None:
231 if root_section is None:
232 root_section = make_section()
233 section = root_section
234 section[name] = option
235
236 charset, includes, root_section = handle_root(root_section)
237 basedir = _os.path.abspath(_os.path.dirname(filename))
238
239 includes = [item.encode(self._charset).decode(charset)
240 for item in includes]
241 if not isinstance(basedir, unicode):
242 fsenc = _sys.getfilesystemencoding()
243 includes = [item.encode(fsenc) for item in includes]
244 oldseen = _included
245 if oldseen is None:
246 oldseen = set()
247 seen = set()
248 for fname in includes:
249 fname = _os.path.normpath(_os.path.join(basedir, fname))
250 rpath = _os.path.realpath(fname)
251 if rpath in oldseen:
252 raise RecursiveIncludeError(filename, lineno, fname)
253 elif rpath not in seen:
254 seen.add(rpath)
255 fp = file(fname, 'rb')
256 try:
257 self.parse(fp, fname, oldseen | seen)
258 finally:
259 fp.close()
260
261 if _included is None:
262 for _, section in config:
263 self._cast(section)
264
265 - def _cast(self, section):
266 """
267 Cast the options of a section to python types
268
269 :Parameters:
270 - `section`: The section to process
271
272 :Types:
273 - `section`: `Section`
274 """
275
276
277 tokre = _re.compile(ur'''
278 [;#]?"[^"\\]*(\\.[^"\\]*)*"
279 | [;#]?'[^'\\]*(\\.[^'\\]*)*'
280 | \S+
281 ''', _re.X).finditer
282 escsub = _re.compile(ur'''(\\(?:
283 x[\da-fA-F]{2}
284 | u[\da-fA-F]{4}
285 | U[\da-fA-F]{8}
286 ))''', _re.X).sub
287 def escsubber(match):
288 """ Substitution function """
289 return match.group(1).encode('ascii').decode('unicode_escape')
290
291 make_option, make_section = self._make_option, self._make_section
292
293 for name, value in section:
294 newvalue = []
295 for match in tokre(u' '.join(value)):
296 val = match.group(0)
297 if val.startswith('#'):
298 continue
299 if (val.startswith(u'"') and val.endswith(u'"')) or \
300 (val.startswith(u"'") and val.endswith(u"'")):
301 val = escsub(escsubber, val[1:-1])
302 else:
303 try:
304 val = human_bool(val)
305 except ValueError:
306 try:
307 val = float(val)
308 except ValueError:
309
310 pass
311 newvalue.append(val)
312 option = make_option(newvalue)
313
314
315 if u'.' in name:
316 parts, sect = name.split(u'.'), section
317 parts.reverse()
318 while parts:
319 part = parts.pop()
320 if parts:
321 if part not in sect:
322 sect[part] = make_section()
323 sect = sect[part]
324 else:
325 sect[part] = option
326 del section[name]
327 else:
328 section[name] = option
329
344
346 """
347 Try to extract a section header from `line`
348
349 :Parameters:
350 - `line`: The line to process
351
352 :Types:
353 - `line`: ``str``
354
355 :return: The section header name or ``None``
356 :rtype: ``str``
357 """
358 if line.startswith(u'['):
359 pos = line.find(u']')
360 if pos > 1:
361 return line[1:pos]
362 return None
363
365 """
366 Parse `line` as option (``name [:=] value``)
367
368 :Parameters:
369 - `line`: The line to process
370
371 :Types:
372 - `line`: ``str``
373
374 :return: The name and the value (both ``None`` if an error occured)
375 :rtype: ``tuple``
376 """
377 pose = line.find('=')
378 posc = line.find(':')
379 pos = min(pose, posc)
380 if pos < 0:
381 pos = max(pose, posc)
382 if pos > 0:
383 return (line[:pos], line[pos + 1:])
384 return (None, None)
385
387 """
388 Make a new `Section` instance
389
390 :return: The new `Section` instance
391 :rtype: `Section`
392 """
393 return Section()
394
396 """
397 Make a new option value
398
399 The function will do the right thing[tm] in order to determine
400 the correct option type based on `valuelist`.
401
402 :Parameters:
403 - `valuelist`: List of values of that option
404
405 :Types:
406 - `valuelist`: ``list``
407
408 :return: Option type appropriate for the valuelist
409 :rtype: any
410 """
411 if not valuelist:
412 valuelist = [None]
413 if len(valuelist) > 1:
414 return valuelist
415 else:
416 return TypedIterOption(valuelist[0])
417
418
420 """ Option, typed dynamically
421
422 Provides an iterator of the single value list
423 """
424
426 """
427 Create the final option type
428
429 This gives the type a new name, inherits from the original type
430 (where possible) and adds an ``__iter__`` method in order to
431 be able to iterate over the one-value-list.
432
433 The following type conversions are done:
434
435 ``bool``
436 Will be converted to ``int``
437
438 :Parameters:
439 - `value`: The value to decorate
440
441 :Types:
442 - `value`: any
443
444 :return: subclass of ``type(value)``
445 :rtype: any
446 """
447 space = {}
448 if value is None:
449 newcls = unicode
450 value = u''
451 def itermethod(self):
452 """ Single value list iteration method """
453
454
455 return iter([])
456 else:
457 newcls = type(value)
458 def itermethod(self):
459 """ Single value list iteration method """
460
461
462 yield value
463 if newcls is bool:
464 newcls = int
465 def reducemethod(self, _cls=cls):
466 """ Mixed Pickles """
467
468 return (_cls, (value,))
469
470 space = dict(
471 __module__=cls.__module__,
472 __iter__=itermethod,
473 __reduce__=reducemethod,
474 )
475 cls = type(cls.__name__, (newcls,), space)
476 return cls(value)
477
478
480 """
481 Config access class
482
483 :IVariables:
484 - `ROOT`: The current working directory at startup time
485
486 :Types:
487 - `ROOT`: ``str``
488 """
489
491 """
492 Initialization
493
494 :Parameters:
495 - `root`: The current working directory at startup time
496
497 :Types:
498 - `root`: ``str``
499 """
500 self.ROOT = root
501 self.__config_sections__ = {}
502
504 """ Return (sectionname, section) tuples of parsed sections """
505 return iter(self.__config_sections__.items())
506
508 """
509 Set a section
510
511 :Parameters:
512 - `name`: Section name
513 - `value`: Section instance
514
515 :Types:
516 - `name`: ``unicode``
517 - `value`: `Section`
518 """
519 self.__config_sections__[unicode(name)] = value
520
522 """
523 Get section by key
524
525 :Parameters:
526 - `name`: The section name
527
528 :Types:
529 - `name`: ``basestring``
530
531 :return: Section object
532 :rtype: `Section`
533
534 :Exceptions:
535 - `KeyError`: section not found
536 """
537 return self.__config_sections__[unicode(name)]
538
540 """
541 Determine if a section named `name` exists
542
543 :Parameters:
544 - `name`: The section name
545
546 :Types:
547 - `name`: ``unicode``
548
549 :return: Does the section exist?
550 :rtype: ``bool``
551 """
552 return unicode(name) in self.__config_sections__
553
555 """
556 Return section by dotted notation
557
558 :Parameters:
559 - `name`: The section name
560
561 :Types:
562 - `name`: ``str``
563
564 :return: Section object
565 :rtype: `Section`
566
567 :Exceptions:
568 - `AttributeError`: section not found
569 """
570 try:
571 return self[name]
572 except KeyError:
573 raise AttributeError(name)
574
575
577 """
578 Config section container
579
580 :IVariables:
581 - `__section_options__`: Option dict
582
583 :Types:
584 - `__section_options__`: ``dict``
585 """
586
588 """ Initialization """
589 self.__section_options__ = {}
590
592 """ (Name, Value) tuple iterator """
593 return iter(self.__section_options__.items())
594
596 """
597 Set a new option
598
599 :Parameters:
600 - `name`: Option name
601 - `value`: Option value
602
603 :Types:
604 - `name`: ``unicode``
605 - `value`: any
606 """
607 self.__section_options__[unicode(name)] = value
608
610 """
611 Return a config option by key
612
613 :Parameters:
614 - `name`: The key to look up
615
616 :Types:
617 - `name`: ``unicode``
618
619 :return: The value of the option
620 :rtype: any
621
622 :Exceptions:
623 - `KeyError`: No suitable option could be found
624 """
625 return self.__section_options__[unicode(name)]
626
628 """
629 Delete option
630
631 :Parameters:
632 - `name`: Option key to process
633
634 :Types:
635 - `name`: ``unicode``
636
637 :Exceptions:
638 - `KeyError`: Option did not exist
639 """
640 del self.__section_options__[unicode(name)]
641
643 """
644 Get option in dotted notation
645
646 :Parameters:
647 - `name`: Option key to look up
648
649 :Types:
650 - `name`: ``str``
651
652 :return: The value of the option
653 :rtype: any
654
655 :Exceptions:
656 - `AttributeError`: No suitable option could be found
657 """
658 try:
659 return self[unicode(name)]
660 except KeyError:
661 raise AttributeError(name)
662
663 - def __call__(self, name, default=None):
664 """
665 Get option or default value
666
667 :Parameters:
668 - `name`: The option key to look up
669 - `default`: Default value
670
671 :Types:
672 - `name`: ``unicode``
673 - `default`: any
674
675 :return: The value of the option
676 :rtype: any
677 """
678 try:
679 return self[unicode(name)]
680 except KeyError:
681 return default
682
684 """
685 Determine whether `name` is an available option key
686
687 :Parameters:
688 - `name`: The option key to look up
689
690 :Types:
691 - `name`: ``unicode``
692
693 :return: Is `name` an available option?
694 :rtype: ``bool``
695 """
696 return unicode(name) in self.__section_options__
697
698
700 """
701 Merge sections together
702
703 :Parameters:
704 `sections` : ``tuple``
705 The sections to merge, later sections take more priority
706
707 :Return: The merged section
708 :Rtype: `Section`
709
710 :Exceptions:
711 - `TypeError`: Either one of the section was not a section or the
712 sections contained unmergable attributes (subsections vs. plain
713 values)
714 """
715 result = Section()
716 for section in sections:
717 if not isinstance(section, Section):
718 raise TypeError("Expected Section, found %r" % (section,))
719 for key, value in dict(section).iteritems():
720 if isinstance(value, Section) and key in result:
721 value = merge_sections(result[key], value)
722 result[key] = value
723 return result
724
725
727 """
728 Interpret human readable boolean value
729
730 ``True``
731 ``yes``, ``true``, ``on``, any number other than ``0``
732 ``False``
733 ``no``, ``false``, ``off``, ``0``, empty, ``none``
734
735 The return value is not a boolean on purpose. It's a number, so you
736 can pass more than just boolean values (by passing a number)
737
738 :Parameters:
739 - `value`: The value to interpret
740
741 :Types:
742 - `value`: ``str``
743
744 :return: ``number``
745 :rtype: ``int``
746 """
747 if not value:
748 value = 0
749 else:
750 self = human_bool
751 value = str(value).lower()
752 if value in self.yes:
753 value = 1
754 elif value in self.no:
755 value = 0
756 else:
757 value = int(value)
758 return value
759
760 human_bool.yes = dict.fromkeys("yes true on 1".split())
761 human_bool.no = dict.fromkeys("no false off 0 none".split())
762
763
764
765 -def dump(config, stream=None):
766 """
767 Dump config object
768
769 :Parameters:
770 `stream` : ``file``
771 The stream to dump to. If omitted or ``None``, it's dumped to
772 ``sys.stdout``.
773 """
774
775
776 def subsection(basename, section):
777 """ Determine option list from subsection """
778 opts = []
779 if basename is None:
780 make_base = lambda s: s
781 else:
782 make_base = lambda s: ".".join((basename, s))
783 for opt_name, opt_value in section:
784 opt_name = make_base(opt_name)
785 if isinstance(opt_value, Section):
786 opts.extend(subsection(opt_name, opt_value))
787 else:
788 opts.append((opt_name, opt_value))
789 return opts
790
791 def pretty(name, value):
792 """ Pretty format a value list """
793 value = tuple(value)
794 if len(value) == 0:
795 return u''
796 elif len(value) == 1:
797 return cast(value[0])
798 result = u" ".join(cast(item) for item in value)
799 if len(u"%s = %s" % (name, result)) < 80:
800 return result
801 return u"\n " + u"\n ".join(cast(item) for item in value)
802
803 def cast(value):
804 """ Format output by type """
805 if isinstance(value, float):
806 return unicode(value)
807 elif isinstance(value, unicode):
808 return u"'%s'" % value.replace(u'\\', u'\\\\').encode(
809 'unicode_escape').decode(
810 'ascii').replace(
811 u"'", u"\\'"
812 )
813 return repr(value).decode('ascii')
814
815 if stream is None:
816 stream = _sys.stdout
817
818 print >> stream, "# ----8<------- WTF config dump -------------"
819 print >> stream, "# This is, what the WTF systems gets to see"
820 print >> stream, "# after loading and merging all config files."
821 print >> stream
822 print >> stream, "charset = %r" % 'utf-8'
823
824 for section_name, section in sorted(config):
825 if section_name is None:
826 continue
827
828 print >> stream
829 print >> stream, (u"[%s]" % section_name).encode('utf-8')
830 for opt_name, opt_value in sorted(subsection(None, section)):
831 print >> stream, u"%s = %s" % (
832 opt_name, pretty(opt_name, opt_value)
833 )
834
835 print >> stream
836 print >> stream, "# ------------- WTF config dump ------->8----"
837
838
839 -def load(name, charset='latin-1'):
840 """
841 Load configuration
842
843 It is not a failure if the file does not exist.
844
845 :Parameters:
846 - `name`: The name of the file
847 - `charset`: Default charset of config files
848
849 :Types:
850 - `name`: ``basestring``
851 - `charset`: ``str``
852
853 :return: A config object
854 :rtype: `Config`
855 """
856 config = Config(_os.path.normpath(_os.path.abspath(_os.getcwd())))
857 parser = Parser(config, charset)
858 try:
859 fp = file(name, 'rb')
860 try:
861 parser.parse(fp, name)
862 finally:
863 fp.close()
864 except IOError:
865 e = _sys.exc_info()
866 try:
867 raise ConfigurationIOError, e[1], e[2]
868 finally:
869 del e
870 return config
871