From 3782b4461fdfe18e19b0fb22cde9dca51f3303b7 Mon Sep 17 00:00:00 2001 From: Alexandre Biancalana Date: Fri, 3 Jun 2016 20:07:03 -0300 Subject: [PATCH 1/8] - add option to build instance name based on openstack metadata on libvirt. --- docs/collectors/LibvirtKVMCollector.md | 2 +- src/collectors/libvirtkvm/libvirtkvm.py | 52 ++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docs/collectors/LibvirtKVMCollector.md b/docs/collectors/LibvirtKVMCollector.md index a49469b71..1a6c30928 100644 --- a/docs/collectors/LibvirtKVMCollector.md +++ b/docs/collectors/LibvirtKVMCollector.md @@ -28,10 +28,10 @@ specific to the compute node | bool uri | qemu:///system | The libvirt connection URI. By default it's
'qemu:///system'. One decent option is
'qemu+unix:///system?socket=/var/run/libvirt/libvit-sock-ro'. | str +format_name_using_metadata | False | Use openstack metadata to generate instance name, its useful when do you and to group VMs metrics by project | string, possible options: ${owner_project},${owner_project_uuid},${instance},${instance_uuid} #### Example Output ``` __EXAMPLESHERE__ ``` - diff --git a/src/collectors/libvirtkvm/libvirtkvm.py b/src/collectors/libvirtkvm/libvirtkvm.py index 9b95fd7c5..bae85fdcc 100644 --- a/src/collectors/libvirtkvm/libvirtkvm.py +++ b/src/collectors/libvirtkvm/libvirtkvm.py @@ -11,6 +11,9 @@ import diamond.collector from diamond.collector import str_to_bool + +from string import Template + try: from xml.etree import ElementTree except ImportError: @@ -21,7 +24,6 @@ except ImportError: libvirt = None - class LibvirtKVMCollector(diamond.collector.Collector): blockStats = { 'read_reqs': 0, @@ -41,6 +43,42 @@ class LibvirtKVMCollector(diamond.collector.Collector): 'tx_drops': 7 } + def format_name_using_metadata(self, dom, format): + s = Template(format) + + tree = ElementTree.fromstring(dom.metadata(2,'http://openstack.org/xmlns/libvirt/nova/1.0')) + + for target in tree.findall('name'): + instance = target.text + + instance_uuid = dom.UUIDString() + + for target in tree.findall('owner/project'): + owner_project_uuid = target.get("uuid") + owner_project = target.text + + for target in tree.findall('owner/user'): + owner_user_uuid = target.get("uuid") + owner_user = target.text + + instance = instance.replace(".","_") + owner_project = owner_project.replace(".","_") + owner_user = owner_user.replace(".","_") + + params = { 'owner_project': owner_project, + 'owner_project_uuid': owner_project_uuid, + 'owner_user': owner_user, + 'owner_user_uuid': owner_user_uuid, + 'instance': instance, + 'instance_uuid': instance_uuid} + + try: + return s.substitute(params) + except KeyError as err: + self.log.error('invalid format parameter %s' % err) + return instance + + def get_default_config_help(self): config_help = super(LibvirtKVMCollector, self).get_default_config_help() @@ -104,10 +142,21 @@ def collect(self): self.log.error('Unable to import libvirt') return {} + self.log.debug('Connecting to %s' % self.config['uri']) + conn = libvirt.openReadOnly(self.config['uri']) + + if conn == None: + self.log.error('Failed to open connection to the hypervisor using %s' % self.config['uri']) + return {} + for dom in [conn.lookupByID(n) for n in conn.listDomainsID()]: if str_to_bool(self.config['sort_by_uuid']): name = dom.UUIDString() + + elif self.config['format_name_using_metadata']: + name = self.format_name_using_metadata(dom, self.config['format_name_using_metadata']) + else: name = dom.name() @@ -163,3 +212,4 @@ def collect(self): self.publish('memory.nominal', mem['actual'] * 1024, instance=name) self.publish('memory.rss', mem['rss'] * 1024, instance=name) + From 6a651a4db8f7a81184643cb47196b32d6b3fb996 Mon Sep 17 00:00:00 2001 From: biancalana Date: Sat, 4 Jun 2016 10:15:50 -0300 Subject: [PATCH 2/8] fix option list --- docs/collectors/LibvirtKVMCollector.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/collectors/LibvirtKVMCollector.md b/docs/collectors/LibvirtKVMCollector.md index 1a6c30928..fed1dfeb1 100644 --- a/docs/collectors/LibvirtKVMCollector.md +++ b/docs/collectors/LibvirtKVMCollector.md @@ -28,7 +28,7 @@ specific to the compute node | bool uri | qemu:///system | The libvirt connection URI. By default it's
'qemu:///system'. One decent option is
'qemu+unix:///system?socket=/var/run/libvirt/libvit-sock-ro'. | str -format_name_using_metadata | False | Use openstack metadata to generate instance name, its useful when do you and to group VMs metrics by project | string, possible options: ${owner_project},${owner_project_uuid},${instance},${instance_uuid} +format_name_using_metadata | False | Use openstack metadata to generate instance name, its useful when do you and to group VMs metrics by project. Available values: ${owner_project},${owner_project_uuid},${instance},${instance_uuid} | list #### Example Output From d4ed9c26ae1c2a030e7f3646a6bdf0b200a677b2 Mon Sep 17 00:00:00 2001 From: biancalana Date: Sat, 4 Jun 2016 10:18:56 -0300 Subject: [PATCH 3/8] remove unneeded html line breaks --- docs/collectors/LibvirtKVMCollector.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/collectors/LibvirtKVMCollector.md b/docs/collectors/LibvirtKVMCollector.md index fed1dfeb1..f537ba48d 100644 --- a/docs/collectors/LibvirtKVMCollector.md +++ b/docs/collectors/LibvirtKVMCollector.md @@ -16,18 +16,13 @@ Uses libvirt to harvest per KVM instance stats Setting | Default | Description | Type --------|---------|-------------|----- byte_unit | byte | Default numeric output(s) | str -cpu_absolute | False | CPU stats reported as percentage by default, or
-as cummulative nanoseconds since VM creation if this is True. | bool +cpu_absolute | False | CPU stats reported as percentage by default, or as cummulative nanoseconds since VM creation if this is True. | bool enabled | False | Enable collecting these metrics | bool measure_collector_time | False | Collect the collector run time in ms | bool metrics_blacklist | None | Regex to match metrics to block. Mutually exclusive with metrics_whitelist | NoneType metrics_whitelist | None | Regex to match metrics to transmit. Mutually exclusive with metrics_blacklist | NoneType -sort_by_uuid | False | Use the of the instance instead of the
- default , useful in Openstack deploments where is only
-specific to the compute node | bool -uri | qemu:///system | The libvirt connection URI. By default it's
-'qemu:///system'. One decent option is
-'qemu+unix:///system?socket=/var/run/libvirt/libvit-sock-ro'. | str +sort_by_uuid | False | Use the of the instance instead of the default , useful in Openstack deploments where is only specific to the compute node | bool +uri | qemu:///system | The libvirt connection URI. By default it's 'qemu:///system'. One decent option is 'qemu+unix:///system?socket=/var/run/libvirt/libvit-sock-ro'. | str format_name_using_metadata | False | Use openstack metadata to generate instance name, its useful when do you and to group VMs metrics by project. Available values: ${owner_project},${owner_project_uuid},${instance},${instance_uuid} | list #### Example Output From 6fddf7fc2b10e27db013b5ce6f4886f066566c18 Mon Sep 17 00:00:00 2001 From: biancalana Date: Sat, 4 Jun 2016 15:35:34 -0300 Subject: [PATCH 4/8] make travis happy --- src/collectors/libvirtkvm/libvirtkvm.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/collectors/libvirtkvm/libvirtkvm.py b/src/collectors/libvirtkvm/libvirtkvm.py index bae85fdcc..18a97ef02 100644 --- a/src/collectors/libvirtkvm/libvirtkvm.py +++ b/src/collectors/libvirtkvm/libvirtkvm.py @@ -46,7 +46,7 @@ class LibvirtKVMCollector(diamond.collector.Collector): def format_name_using_metadata(self, dom, format): s = Template(format) - tree = ElementTree.fromstring(dom.metadata(2,'http://openstack.org/xmlns/libvirt/nova/1.0')) + tree = ElementTree.fromstring(dom.metadata(2, 'http://openstack.org/xmlns/libvirt/nova/1.0')) for target in tree.findall('name'): instance = target.text @@ -61,11 +61,11 @@ def format_name_using_metadata(self, dom, format): owner_user_uuid = target.get("uuid") owner_user = target.text - instance = instance.replace(".","_") - owner_project = owner_project.replace(".","_") - owner_user = owner_user.replace(".","_") + instance = instance.replace(".", "_") + owner_project = owner_project.replace(".", "_") + owner_user = owner_user.replace(".", "_") - params = { 'owner_project': owner_project, + params = { 'owner_project': owner_project, 'owner_project_uuid': owner_project_uuid, 'owner_user': owner_user, 'owner_user_uuid': owner_user_uuid, @@ -78,7 +78,6 @@ def format_name_using_metadata(self, dom, format): self.log.error('invalid format parameter %s' % err) return instance - def get_default_config_help(self): config_help = super(LibvirtKVMCollector, self).get_default_config_help() @@ -146,7 +145,7 @@ def collect(self): conn = libvirt.openReadOnly(self.config['uri']) - if conn == None: + if conn is None: self.log.error('Failed to open connection to the hypervisor using %s' % self.config['uri']) return {} @@ -155,7 +154,8 @@ def collect(self): name = dom.UUIDString() elif self.config['format_name_using_metadata']: - name = self.format_name_using_metadata(dom, self.config['format_name_using_metadata']) + name = self.format_name_using_metadata(dom, + self.config['format_name_using_metadata']) else: name = dom.name() @@ -212,4 +212,3 @@ def collect(self): self.publish('memory.nominal', mem['actual'] * 1024, instance=name) self.publish('memory.rss', mem['rss'] * 1024, instance=name) - From ab970f24f4a18d527bf11e4a425c89cad6fa4acd Mon Sep 17 00:00:00 2001 From: biancalana Date: Sat, 4 Jun 2016 16:35:57 -0300 Subject: [PATCH 5/8] make travis happy, really --- src/collectors/libvirtkvm/libvirtkvm.py | 29 +++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/collectors/libvirtkvm/libvirtkvm.py b/src/collectors/libvirtkvm/libvirtkvm.py index 18a97ef02..f2608dc5a 100644 --- a/src/collectors/libvirtkvm/libvirtkvm.py +++ b/src/collectors/libvirtkvm/libvirtkvm.py @@ -24,6 +24,7 @@ except ImportError: libvirt = None + class LibvirtKVMCollector(diamond.collector.Collector): blockStats = { 'read_reqs': 0, @@ -46,7 +47,9 @@ class LibvirtKVMCollector(diamond.collector.Collector): def format_name_using_metadata(self, dom, format): s = Template(format) - tree = ElementTree.fromstring(dom.metadata(2, 'http://openstack.org/xmlns/libvirt/nova/1.0')) + tree = ElementTree.fromstring(dom.metadata( + 2, + "http://openstack.org/xmlns/libvirt/nova/1.0")) for target in tree.findall('name'): instance = target.text @@ -61,16 +64,16 @@ def format_name_using_metadata(self, dom, format): owner_user_uuid = target.get("uuid") owner_user = target.text - instance = instance.replace(".", "_") + instance = instance.replace(".", "_") owner_project = owner_project.replace(".", "_") - owner_user = owner_user.replace(".", "_") + owner_user = owner_user.replace(".", "_") - params = { 'owner_project': owner_project, - 'owner_project_uuid': owner_project_uuid, - 'owner_user': owner_user, - 'owner_user_uuid': owner_user_uuid, - 'instance': instance, - 'instance_uuid': instance_uuid} + params = {'owner_project': owner_project, + 'owner_project_uuid': owner_project_uuid, + 'owner_user': owner_user, + 'owner_user_uuid': owner_user_uuid, + 'instance': instance, + 'instance_uuid': instance_uuid} try: return s.substitute(params) @@ -146,7 +149,8 @@ def collect(self): conn = libvirt.openReadOnly(self.config['uri']) if conn is None: - self.log.error('Failed to open connection to the hypervisor using %s' % self.config['uri']) + self.log.error('Failed to open connection to the hypervisor on %s' % + self.config['uri']) return {} for dom in [conn.lookupByID(n) for n in conn.listDomainsID()]: @@ -154,8 +158,9 @@ def collect(self): name = dom.UUIDString() elif self.config['format_name_using_metadata']: - name = self.format_name_using_metadata(dom, - self.config['format_name_using_metadata']) + name = self.format_name_using_metadata( + dom, + self.config['format_name_using_metadata']) else: name = dom.name() From cf1f23738b96373282c446c5cb31f25307aac8d0 Mon Sep 17 00:00:00 2001 From: biancalana Date: Sat, 4 Jun 2016 16:42:15 -0300 Subject: [PATCH 6/8] make travis happy, really --- src/collectors/libvirtkvm/libvirtkvm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/collectors/libvirtkvm/libvirtkvm.py b/src/collectors/libvirtkvm/libvirtkvm.py index f2608dc5a..6316305ad 100644 --- a/src/collectors/libvirtkvm/libvirtkvm.py +++ b/src/collectors/libvirtkvm/libvirtkvm.py @@ -47,9 +47,9 @@ class LibvirtKVMCollector(diamond.collector.Collector): def format_name_using_metadata(self, dom, format): s = Template(format) - tree = ElementTree.fromstring(dom.metadata( - 2, - "http://openstack.org/xmlns/libvirt/nova/1.0")) + tree = ElementTree.fromstring( + dom.metadata(2, + "http://openstack.org/xmlns/libvirt/nova/1.0")) for target in tree.findall('name'): instance = target.text @@ -159,8 +159,8 @@ def collect(self): elif self.config['format_name_using_metadata']: name = self.format_name_using_metadata( - dom, - self.config['format_name_using_metadata']) + dom, + self.config['format_name_using_metadata']) else: name = dom.name() From 6aacb20110981419c478576db650ea2c90c48f8d Mon Sep 17 00:00:00 2001 From: biancalana Date: Tue, 7 Jun 2016 14:04:11 -0300 Subject: [PATCH 7/8] initial tests --- .../libvirtkvm/test/fixtures/libvirt_metadata | 0 .../libvirtkvm/test/testlibvirtkvm.py | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/collectors/libvirtkvm/test/fixtures/libvirt_metadata diff --git a/src/collectors/libvirtkvm/test/fixtures/libvirt_metadata b/src/collectors/libvirtkvm/test/fixtures/libvirt_metadata new file mode 100644 index 000000000..e69de29bb diff --git a/src/collectors/libvirtkvm/test/testlibvirtkvm.py b/src/collectors/libvirtkvm/test/testlibvirtkvm.py index e377a4587..daf8f1248 100644 --- a/src/collectors/libvirtkvm/test/testlibvirtkvm.py +++ b/src/collectors/libvirtkvm/test/testlibvirtkvm.py @@ -21,6 +21,40 @@ def setUp(self): def test_import(self): self.assertTrue(LibvirtKVMCollector) + @patch.object(Collector, 'publish') + + def test_should_work_with_real_data(self, publish_mock, conn_mock): + LibvirtKVMCollector.conn = Mock() + conn = True + listDomainsID = [ 1 ] + lookupByID = True + + LibvirtKVMCollector.conn_mock = Mock() + + dom.getCPUStats + dom.getCPUStats + + LibvirtKVMCollector.dom. = self.getFixturePath('proc_meminfo') + self.collector.collect() + + metrics = { + 'MemTotal': 49554212, + 'MemFree': 35194496, + 'Buffers': 1526304, + 'Cached': 10726736, + 'Active': 10022168, + 'Dirty': 24748, + 'Inactive': 2524928, + 'Shmem': 276, + 'SwapTotal': 262143996, + 'SwapFree': 262143996, + 'SwapCached': 0, + 'VmallocTotal': 34359738367, + 'VmallocUsed': 445452, + 'VmallocChunk': 34311049240 + } + + ############################################################################### if __name__ == "__main__": unittest.main() From 1078a788947640c1337e2ed930ed29491066e67d Mon Sep 17 00:00:00 2001 From: Alexandre Biancalana Date: Tue, 7 Jun 2016 19:18:48 -0300 Subject: [PATCH 8/8] - added some mocking --- .../libvirtkvm/test/testlibvirtkvm.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/collectors/libvirtkvm/test/testlibvirtkvm.py b/src/collectors/libvirtkvm/test/testlibvirtkvm.py index daf8f1248..ba1c6e46c 100644 --- a/src/collectors/libvirtkvm/test/testlibvirtkvm.py +++ b/src/collectors/libvirtkvm/test/testlibvirtkvm.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest +from mock import Mock +from mock import patch from libvirtkvm import LibvirtKVMCollector - ############################################################################### class TestLibvirtKVMCollector(CollectorTestCase): @@ -21,20 +22,27 @@ def setUp(self): def test_import(self): self.assertTrue(LibvirtKVMCollector) + @patch('libvirt') @patch.object(Collector, 'publish') - def test_should_work_with_real_data(self, publish_mock, conn_mock): - LibvirtKVMCollector.conn = Mock() - conn = True + def test_should_work_with_real_data(self, publish_mock, libvirt_mock): + libvirt = Mock() + conn_mock = Mock() + dom_mock = Mock() + listDomainsID = [ 1 ] - lookupByID = True - LibvirtKVMCollector.conn_mock = Mock() + libvirt_mock.return_value = libvirt + libvirt.openReadOnly.return_value = conn_mock + conn_mock.listDomainsID.return_value = listDomainsID + conn_mock.lookupByID.return_value = dom_mock + + dom_mock.UUIDString.return_value = "uuid" - dom.getCPUStats - dom.getCPUStats + dom_mock.getCPUStats.return_value = [{'cpu_time': 30375130070L, + 'system_time': 6980000000L, + 'user_time': 3120000000L}] - LibvirtKVMCollector.dom. = self.getFixturePath('proc_meminfo') self.collector.collect() metrics = {