- raise ConfigurationError when there is the xml.dom runs into a xml
[myplc.git] / plc_config.py
1 #!/usr/bin/python
2 #
3 # Merge PlanetLab Central (PLC) configuration files into a variety of
4 # output formats. These files represent the global configuration for a
5 # PLC installation.
6 #
7 # Mark Huang <mlhuang@cs.princeton.edu>
8 # Copyright (C) 2006 The Trustees of Princeton University
9 #
10 # $Id$
11 #
12
13 import xml.dom.minidom
14 from xml.parsers.expat import ExpatError
15 from StringIO import StringIO
16 import time
17 import re
18 import textwrap
19 import codecs
20 import os
21 import types
22
23
24 class ConfigurationException(Exception): pass
25
26 class PLCConfiguration:
27     """
28     Configuration file store. Optionally instantiate with a file path
29     or object:
30
31     plc = PLCConfiguration()
32     plc = PLCConfiguration(fileobj)
33     plc = PLCConfiguration("/etc/planetlab/plc_config.xml")
34
35     You may load() additional files later, which will be merged into
36     the current configuration:
37
38     plc.load("/etc/planetlab/local.xml")
39
40     You may also save() the configuration. If a file path or object is
41     not specified, the configuration will be written to the file path
42     or object that was first loaded.
43     
44     plc.save()
45     plc.save("/etc/planetlab/plc_config.xml")
46     """
47
48     def __init__(self, file = None):
49         impl = xml.dom.minidom.getDOMImplementation()
50         self._dom = impl.createDocument(None, "configuration", None)
51         self._variables = {}
52         self._packages = {}
53         self._files = []
54
55         if file is not None:
56             self.load(file)
57
58
59     def _get_text(self, node):
60         """
61         Get the text of a text node.
62         """
63
64         if node.firstChild and \
65            node.firstChild.nodeType == node.TEXT_NODE:
66             if node.firstChild.data is None:
67                 # Interpret simple presence of node as "", not NULL
68                 return ""
69             else:
70                 return node.firstChild.data
71
72         return None
73
74
75     def _get_text_of_child(self, parent, name):
76         """
77         Get the text of a (direct) child text node.
78         """
79
80         for node in parent.childNodes:
81             if node.nodeType == node.ELEMENT_NODE and \
82                node.tagName == name:
83                 return self._get_text(node)
84
85         return None
86
87
88     def _set_text(self, node, data):
89         """
90         Set the text of a text node.
91         """
92
93         if node.firstChild and \
94            node.firstChild.nodeType == node.TEXT_NODE:
95             if data is None:
96                 node.removeChild(node.firstChild)
97             else:
98                 node.firstChild.data = data
99         elif data is not None:
100             text = TrimText()
101             text.data = data
102             node.appendChild(text)
103
104
105     def _set_text_of_child(self, parent, name, data):
106         """
107         Set the text of a (direct) child text node.
108         """
109
110         for node in parent.childNodes:
111             if node.nodeType == node.ELEMENT_NODE and \
112                node.tagName == name:
113                 self._set_text(node, data)
114                 return
115
116         child = TrimTextElement(name)
117         self._set_text(child, data)
118         parent.appendChild(child)
119
120
121     def _category_element_to_dict(self, category_element):
122         """
123         Turn a <category> element into a dictionary of its attributes
124         and child text nodes.
125         """
126
127         category = {}
128         category['id'] = category_element.getAttribute('id').lower()
129         for node in category_element.childNodes:
130             if node.nodeType == node.ELEMENT_NODE and \
131                node.tagName in ['name', 'description']:
132                 category[node.tagName] = self._get_text_of_child(category_element, node.tagName)
133         category['element'] = category_element
134
135         return category
136
137
138     def _variable_element_to_dict(self, variable_element):
139         """
140         Turn a <variable> element into a dictionary of its attributes
141         and child text nodes.
142         """
143
144         variable = {}
145         variable['id'] = variable_element.getAttribute('id').lower()
146         if variable_element.hasAttribute('type'):
147             variable['type'] = variable_element.getAttribute('type')
148         for node in variable_element.childNodes:
149             if node.nodeType == node.ELEMENT_NODE and \
150                node.tagName in ['name', 'value', 'description']:
151                 variable[node.tagName] = self._get_text_of_child(variable_element, node.tagName)
152         variable['element'] = variable_element
153
154         return variable
155
156
157     def _group_element_to_dict(self, group_element):
158         """
159         Turn a <group> element into a dictionary of its attributes
160         and child text nodes.
161         """
162
163         group = {}
164         for node in group_element.childNodes:
165             if node.nodeType == node.ELEMENT_NODE and \
166                node.tagName in ['id', 'name', 'default', 'description', 'uservisible']:
167                 group[node.tagName] = self._get_text_of_child(group_element, node.tagName)
168         group['element'] = group_element
169
170         return group
171
172
173     def _packagereq_element_to_dict(self, packagereq_element):
174         """
175         Turns a <packagereq> element into a dictionary of its attributes
176         and child text nodes.
177         """
178
179         package = {}
180         if packagereq_element.hasAttribute('type'):
181             package['type'] = packagereq_element.getAttribute('type')
182         package['name'] = self._get_text(packagereq_element)
183         package['element'] = packagereq_element
184
185         return package
186
187
188     def load(self, file = "/etc/planetlab/plc_config.xml"):
189         """
190         Merge file into configuration store.
191         """
192
193         try:
194             dom = xml.dom.minidom.parse(file)
195         except ExpatError, e:
196             raise ConfigurationException, e
197
198         if type(file) in types.StringTypes:
199             self._files.append(os.path.abspath(file))
200
201         # Parse <variables> section
202         for variables_element in dom.getElementsByTagName('variables'):
203             for category_element in variables_element.getElementsByTagName('category'):
204                 category = self._category_element_to_dict(category_element)
205                 self.set(category, None)
206
207                 for variablelist_element in category_element.getElementsByTagName('variablelist'):
208                     for variable_element in variablelist_element.getElementsByTagName('variable'):
209                         variable = self._variable_element_to_dict(variable_element)
210                         self.set(category, variable)
211
212         # Parse <comps> section
213         for comps_element in dom.getElementsByTagName('comps'):
214             for group_element in comps_element.getElementsByTagName('group'):
215                 group = self._group_element_to_dict(group_element)
216                 self.add_package(group, None)
217
218                 for packagereq_element in group_element.getElementsByTagName('packagereq'):
219                     package = self._packagereq_element_to_dict(packagereq_element)
220                     self.add_package(group, package)
221
222
223     def save(self, file = None):
224         """
225         Write configuration store to file.
226         """
227
228         if file is None:
229             if self._files:
230                 file = self._files[0]
231             else:
232                 file = "/etc/planetlab/plc_config.xml"
233
234         if type(file) in types.StringTypes:
235             fileobj = open(file, 'w')
236         else:
237             fileobj = file
238
239         fileobj.seek(0)
240         fileobj.write(self.output_xml())
241         fileobj.truncate()
242
243         fileobj.close()
244
245     def verify(self, default, read, verify_variables={}):
246         """ Confirm that the existing configuration is consistent
247             according to the checks below.
248
249             It looks for filled-in values in the order of, local object (self),
250             followed by cread (read values), and finally default values.
251
252         Arguments: 
253
254             default configuration
255             site configuration
256             list of category/variable tuples to validate in these configurations
257
258         Returns:
259
260             dict of values for the category/variables passed in
261             If an exception is found, ConfigurationException is raised.
262
263         """
264
265         validated_variables = {}
266         for category_id, variable_id in verify_variables.iteritems():
267             category_id = category_id.lower()
268             variable_id = variable_id.lower()
269             variable_value = None
270             sources = (self, read, default)
271             for source in sources:
272                 (category_value, variable_value) = source.get(category_id,variable_id)
273                 if variable_value <> None:
274                     entry = validated_variables.get(category_id,[])
275                     entry.append(variable_value['value'])
276                     validated_variables["%s_%s"%(category_id.upper(),variable_id.upper())]=entry
277                     break
278             if variable_value == None:
279                 raise ConfigurationException("Cannot find %s_%s)" % \
280                                              (category_id.upper(),
281                                               variable_id.upper()))
282         return validated_variables
283
284     def get(self, category_id, variable_id):
285         """
286         Get the specified variable in the specified category.
287
288         Arguments:
289
290         category_id = unique category identifier (e.g., 'plc_www')
291         variable_id = unique variable identifier (e.g., 'port')
292
293         Returns:
294
295         variable = { 'id': "variable_identifier",
296                      'type': "variable_type",
297                      'value': "variable_value",
298                      'name': "Variable name",
299                      'description': "Variable description" }
300         """
301
302         if self._variables.has_key(category_id.lower()):
303             (category, variables) = self._variables[category_id]
304             if variables.has_key(variable_id.lower()):
305                 variable = variables[variable_id]
306             else:
307                 variable = None
308         else:
309             category = None
310             variable = None
311
312         return (category, variable)
313
314
315     def delete(self, category_id, variable_id):
316         """
317         Delete the specified variable from the specified category. If
318         variable_id is None, deletes all variables from the specified
319         category as well as the category itself.
320
321         Arguments:
322
323         category_id = unique category identifier (e.g., 'plc_www')
324         variable_id = unique variable identifier (e.g., 'port')
325         """
326
327         if self._variables.has_key(category_id.lower()):
328             (category, variables) = self._variables[category_id]
329             if variable_id is None:
330                 category['element'].parentNode.removeChild(category['element'])
331                 del self._variables[category_id]
332             elif variables.has_key(variable_id.lower()):
333                 variable = variables[variable_id]
334                 variable['element'].parentNode.removeChild(variable['element'])
335                 del variables[variable_id]
336
337
338     def set(self, category, variable):
339         """
340         Add and/or update the specified variable. The 'id' fields are
341         mandatory. If a field is not specified and the category and/or
342         variable already exists, the field will not be updated. If
343         'variable' is None, only adds and/or updates the specified
344         category.
345
346         Arguments:
347
348         category = { 'id': "category_identifier",
349                      'name': "Category name",
350                      'description': "Category description" }
351
352         variable = { 'id': "variable_identifier",
353                      'type': "variable_type",
354                      'value': "variable_value",
355                      'name': "Variable name",
356                      'description': "Variable description" }
357         """
358
359         if not category.has_key('id') or type(category['id']) not in types.StringTypes:
360             return
361         
362         category_id = category['id'].lower()
363
364         if self._variables.has_key(category_id):
365             # Existing category
366             (old_category, variables) = self._variables[category_id]
367
368             # Merge category attributes
369             for tag in ['name', 'description']:
370                 if category.has_key(tag):
371                     old_category[tag] = category[tag]
372                     self._set_text_of_child(old_category['element'], tag, category[tag])
373
374             category_element = old_category['element']
375         else:
376             # Merge into DOM
377             category_element = self._dom.createElement('category')
378             category_element.setAttribute('id', category_id)
379             for tag in ['name', 'description']:
380                 if category.has_key(tag):
381                     self._set_text_of_child(category_element, tag, category[tag])
382
383             if self._dom.documentElement.getElementsByTagName('variables'):
384                 variables_element = self._dom.documentElement.getElementsByTagName('variables')[0]
385             else:
386                 variables_element = self._dom.createElement('variables')
387                 self._dom.documentElement.appendChild(variables_element)
388             variables_element.appendChild(category_element)
389
390             # Cache it
391             category['element'] = category_element
392             variables = {}
393             self._variables[category_id] = (category, variables)
394
395         if variable is None or not variable.has_key('id') or type(variable['id']) not in types.StringTypes:
396             return
397
398         variable_id = variable['id'].lower()
399
400         if variables.has_key(variable_id):
401             # Existing variable
402             old_variable = variables[variable_id]
403
404             # Merge variable attributes
405             for attribute in ['type']:
406                 if variable.has_key(attribute):
407                     old_variable[attribute] = variable[attribute]
408                     old_variable['element'].setAttribute(attribute, variable[attribute])
409             for tag in ['name', 'value', 'description']:
410                 if variable.has_key(tag):
411                     old_variable[tag] = variable[tag]
412                     self._set_text_of_child(old_variable['element'], tag, variable[tag])
413         else:
414             # Merge into DOM
415             variable_element = self._dom.createElement('variable')
416             variable_element.setAttribute('id', variable_id)
417             for attribute in ['type']:
418                 if variable.has_key(attribute):
419                     variable_element.setAttribute(attribute, variable[attribute])
420             for tag in ['name', 'value', 'description']:
421                 if variable.has_key(tag):
422                     self._set_text_of_child(variable_element, tag, variable[tag])
423                 
424             if category_element.getElementsByTagName('variablelist'):
425                 variablelist_element = category_element.getElementsByTagName('variablelist')[0]
426             else:
427                 variablelist_element = self._dom.createElement('variablelist')
428                 category_element.appendChild(variablelist_element)
429             variablelist_element.appendChild(variable_element)
430
431             # Cache it
432             variable['element'] = variable_element
433             variables[variable_id] = variable
434
435
436     def locate_varname (self, varname):
437         """
438         Locates category and variable from a variable's (shell) name
439
440         Returns:
441         (variable, category) when found
442         (None, None) otherwise
443         """
444         
445         for (category_id, (category, variables)) in self._variables.iteritems():
446             for variable in variables.values():
447                 (id, name, value, comments) = self._sanitize_variable(category_id, variable)
448                 if (id == varname):
449                     return (category,variable)
450         return (None,None)
451
452     def get_package(self, group_id, package_name):
453         """
454         Get the specified package in the specified package group.
455
456         Arguments:
457
458         group_id - unique group id (e.g., 'plc')
459         package_name - unique package name (e.g., 'postgresql')
460
461         Returns:
462
463         package = { 'name': "package_name",
464                     'type': "mandatory|optional" }
465         """
466
467         if self._packages.has_key(group_id.lower()):
468             (group, packages) = self._packages[group_id]
469             if packages.has_key(package_name):
470                 package = packages[package_name]
471             else:
472                 package = None
473         else:
474             group = None
475             package = None
476
477         return (group, package)
478
479
480     def delete_package(self, group_id, package_name):
481         """
482         Deletes the specified variable from the specified category. If
483         variable_id is None, deletes all variables from the specified
484         category as well as the category itself.
485
486         Arguments:
487
488         group_id - unique group id (e.g., 'plc')
489         package_name - unique package name (e.g., 'postgresql')
490         """
491
492         if self._packages.has_key(group_id):
493             (group, packages) = self._packages[group_id]
494             if package_name is None:
495                 group['element'].parentNode.removeChild(group['element'])
496                 del self._packages[group_id]
497             elif packages.has_key(package_name.lower()):
498                 package = packages[package_name]
499                 package['element'].parentNode.removeChild(package['element'])
500                 del packages[package_name]
501
502
503     def add_package(self, group, package):
504         """
505         Add and/or update the specified package. The 'id' and 'name'
506         fields are mandatory. If a field is not specified and the
507         package or group already exists, the field will not be
508         updated. If package is None, only adds/or updates the
509         specified group.
510
511         Arguments:
512
513         group = { 'id': "group_identifier",
514                   'name': "Group name",
515                   'default': "true|false",
516                   'description': "Group description",
517                   'uservisible': "true|false" }
518
519         package = { 'name': "package_name",
520                     'type': "mandatory|optional" }
521         """
522
523         if not group.has_key('id'):
524             return
525
526         group_id = group['id']
527
528         if self._packages.has_key(group_id):
529             # Existing group
530             (old_group, packages) = self._packages[group_id]
531
532             # Merge group attributes
533             for tag in ['id', 'name', 'default', 'description', 'uservisible']:
534                 if group.has_key(tag):
535                     old_group[tag] = group[tag]
536                     self._set_text_of_child(old_group['element'], tag, group[tag])
537
538             group_element = old_group['element']
539         else:
540             # Merge into DOM
541             group_element = self._dom.createElement('group')
542             for tag in ['id', 'name', 'default', 'description', 'uservisible']:
543                 if group.has_key(tag):
544                     self._set_text_of_child(group_element, tag, group[tag])
545
546             if self._dom.documentElement.getElementsByTagName('comps'):
547                 comps_element = self._dom.documentElement.getElementsByTagName('comps')[0]
548             else:
549                 comps_element = self._dom.createElement('comps')
550                 self._dom.documentElement.appendChild(comps_element)
551             comps_element.appendChild(group_element)
552
553             # Cache it
554             group['element'] = group_element
555             packages = {}
556             self._packages[group_id] = (group, packages)
557
558         if package is None or not package.has_key('name'):
559             return
560
561         package_name = package['name']
562         if packages.has_key(package_name):
563             # Existing package
564             old_package = packages[package_name]
565
566             # Merge variable attributes
567             for attribute in ['type']:
568                 if package.has_key(attribute):
569                     old_package[attribute] = package[attribute]
570                     old_package['element'].setAttribute(attribute, package[attribute])
571         else:
572             # Merge into DOM
573             packagereq_element = TrimTextElement('packagereq')
574             self._set_text(packagereq_element, package_name)
575             for attribute in ['type']:
576                 if package.has_key(attribute):
577                     packagereq_element.setAttribute(attribute, package[attribute])
578                 
579             if group_element.getElementsByTagName('packagelist'):
580                 packagelist_element = group_element.getElementsByTagName('packagelist')[0]
581             else:
582                 packagelist_element = self._dom.createElement('packagelist')
583                 group_element.appendChild(packagelist_element)
584             packagelist_element.appendChild(packagereq_element)
585
586             # Cache it
587             package['element'] = packagereq_element
588             packages[package_name] = package
589
590
591     def variables(self):
592         """
593         Return all variables.
594
595         Returns:
596
597         variables = { 'category_id': (category, variablelist) }
598
599         category = { 'id': "category_identifier",
600                      'name': "Category name",
601                      'description': "Category description" }
602
603         variablelist = { 'variable_id': variable }
604
605         variable = { 'id': "variable_identifier",
606                      'type': "variable_type",
607                      'value': "variable_value",
608                      'name': "Variable name",
609                      'description': "Variable description" }
610         """
611
612         return self._variables
613
614
615     def packages(self):
616         """
617         Return all packages.
618
619         Returns:
620
621         packages = { 'group_id': (group, packagelist) }
622
623         group = { 'id': "group_identifier",
624                   'name': "Group name",
625                   'default': "true|false",
626                   'description': "Group description",
627                   'uservisible': "true|false" }
628
629         packagelist = { 'package_name': package }
630
631         package = { 'name': "package_name",
632                     'type': "mandatory|optional" }
633         """
634
635         return self._packages
636
637
638     def _sanitize_variable(self, category_id, variable):
639         assert variable.has_key('id')
640         # Prepend variable name with category label
641         id = category_id + "_" + variable['id']
642         # And uppercase it
643         id = id.upper()
644
645         if variable.has_key('type'):
646             type = variable['type']
647         else:
648             type = None
649
650         if variable.has_key('name'):
651             name = variable['name']
652         else:
653             name = None
654
655         if variable.has_key('value') and variable['value'] is not None:
656             value = variable['value']
657             if type == "int" or type == "double":
658                 # bash, Python, and PHP do not require that numbers be quoted
659                 pass
660             elif type == "boolean":
661                 # bash, Python, and PHP can all agree on 0 and 1
662                 if value == "true":
663                     value = "1"
664                 else:
665                     value = "0"
666             else:
667                 # bash, Python, and PHP all support strong single quoting
668                 value = "'" + value.replace("'", "\\'") + "'"
669         else:
670             value = None
671
672         if variable.has_key('description') and variable['description'] is not None:
673             description = variable['description']
674             # Collapse consecutive whitespace
675             description = re.sub(r'\s+', ' ', description)
676             # Wrap comments at 70 columns
677             wrapper = textwrap.TextWrapper()
678             comments = wrapper.wrap(description)
679         else:
680             comments = None
681
682         return (id, name, value, comments)
683
684
685     def _header(self):
686         header = """
687 DO NOT EDIT. This file was automatically generated at
688 %s from:
689
690 %s
691 """ % (time.asctime(), os.linesep.join(self._files))
692
693         # Get rid of the surrounding newlines
694         return header.strip().split(os.linesep)
695
696
697     def output_shell(self, show_comments = True, encoding = "utf-8"):
698         """
699         Return variables as a shell script.
700         """
701
702         buf = codecs.lookup(encoding)[3](StringIO())
703         buf.writelines(["# " + line + os.linesep for line in self._header()])
704
705         for (category_id, (category, variables)) in self._variables.iteritems():
706             for variable in variables.values():
707                 (id, name, value, comments) = self._sanitize_variable(category_id, variable)
708                 if show_comments:
709                     buf.write(os.linesep)
710                     if name is not None:
711                         buf.write("# " + name + os.linesep)
712                     if comments is not None:
713                         buf.writelines(["# " + line + os.linesep for line in comments])
714                 # bash does not have the concept of NULL
715                 if value is not None:
716                     buf.write(id + "=" + value + os.linesep)
717
718         return buf.getvalue()
719
720
721     def output_php(self, encoding = "utf-8"):
722         """
723         Return variables as a PHP script.
724         """
725
726         buf = codecs.lookup(encoding)[3](StringIO())
727         buf.write("<?php" + os.linesep)
728         buf.writelines(["// " + line + os.linesep for line in self._header()])
729
730         for (category_id, (category, variables)) in self._variables.iteritems():
731             for variable in variables.values():
732                 (id, name, value, comments) = self._sanitize_variable(category_id, variable)
733                 buf.write(os.linesep)
734                 if name is not None:
735                     buf.write("// " + name + os.linesep)
736                 if comments is not None:
737                     buf.writelines(["// " + line + os.linesep for line in comments])
738                 if value is None:
739                     value = 'NULL'
740                 buf.write("define('%s', %s);" % (id, value) + os.linesep)
741
742         buf.write("?>" + os.linesep)
743
744         return buf.getvalue()
745
746
747     def output_xml(self, encoding = "utf-8"):
748         """
749         Return variables in original XML format.
750         """
751
752         buf = codecs.lookup(encoding)[3](StringIO())
753         self._dom.writexml(buf, addindent = "  ", indent = "", newl = "\n", encoding = encoding)
754
755         return buf.getvalue()
756
757
758     def output_variables(self, encoding = "utf-8"):
759         """
760         Return list of all variable names.
761         """
762
763         buf = codecs.lookup(encoding)[3](StringIO())
764
765         for (category_id, (category, variables)) in self._variables.iteritems():
766             for variable in variables.values():
767                 (id, name, value, comments) = self._sanitize_variable(category_id, variable)
768                 buf.write(id + os.linesep)
769
770         return buf.getvalue()
771
772
773     def output_packages(self, encoding = "utf-8"):
774         """
775         Return list of all packages.
776         """
777
778         buf = codecs.lookup(encoding)[3](StringIO())
779
780         for (group, packages) in self._packages.values():
781             buf.write(os.linesep.join(packages.keys()))
782
783         if buf.tell():
784             buf.write(os.linesep)
785
786         return buf.getvalue()
787
788
789     def output_groups(self, encoding = "utf-8"):
790         """
791         Return list of all package group names.
792         """
793
794         buf = codecs.lookup(encoding)[3](StringIO())
795
796         for (group, packages) in self._packages.values():
797             buf.write(group['name'] + os.linesep)
798
799         return buf.getvalue()
800
801
802     def output_comps(self, encoding = "utf-8"):
803         """
804         Return <comps> section of configuration.
805         """
806
807         if self._dom is None or \
808            not self._dom.getElementsByTagName("comps"):
809             return
810         comps = self._dom.getElementsByTagName("comps")[0]
811
812         impl = xml.dom.minidom.getDOMImplementation()
813         doc = impl.createDocument(None, "comps", None)
814
815         buf = codecs.lookup(encoding)[3](StringIO())
816
817         # Pop it off the DOM temporarily
818         parent = comps.parentNode
819         parent.removeChild(comps)
820
821         doc.replaceChild(comps, doc.documentElement)
822         doc.writexml(buf, encoding = encoding)
823
824         # Put it back
825         parent.appendChild(comps)
826
827         return buf.getvalue()
828
829
830 # xml.dom.minidom.Text.writexml adds surrounding whitespace to textual
831 # data when pretty-printing. Override this behavior.
832 class TrimText(xml.dom.minidom.Text):
833     def writexml(self, writer, indent="", addindent="", newl=""):
834         xml.dom.minidom.Text.writexml(self, writer, "", "", "")
835
836
837 class TrimTextElement(xml.dom.minidom.Element):
838     def writexml(self, writer, indent="", addindent="", newl=""):
839         writer.write(indent)
840         xml.dom.minidom.Element.writexml(self, writer, "", "", "")
841         writer.write(newl)
842
843
844 if __name__ == '__main__':
845     import sys
846     if len(sys.argv) > 1 and sys.argv[1] in ['build', 'install', 'uninstall']:
847         from distutils.core import setup
848         setup(py_modules=["plc_config"])