本帖最后由 marsaber 于 2015-1-13 14:54 编辑
Unable to connect to AMQP server?
控制节点的qpid服务有问题?
[root@compute1 ~]# tail -f /var/log/nova/compute.log
2015-01-13 14:41:30.205 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Failed to consume message from queue: connection aborted
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid Traceback (most recent call last):
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 546, in ensure
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid return method(*args, **kwargs)
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 599, in _consume
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid nxt_receiver = self.session.next_receiver(timeout=timeout)
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "<string>", line 6, in next_receiver
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 665, in next_receiver
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid if self._ecwait(lambda: self.incoming, timeout):
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 50, in _ecwait
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid result = self._ewait(lambda: self.closed or predicate(), timeout)
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 571, in _ewait
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid result = self.connection._ewait(lambda: self.error or predicate(), timeout)
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 214, in _ewait
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid self.check_error()
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 207, in check_error
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid raise self.error
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid ConnectionError: connection aborted
2015-01-13 14:41:30.205 1956 TRACE oslo.messaging._drivers.impl_qpid
2015-01-13 14:41:30.292 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 1 seconds
2015-01-13 14:41:30.298 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Failed to consume message from queue: connection aborted
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid Traceback (most recent call last):
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 546, in ensure
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid return method(*args, **kwargs)
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 599, in _consume
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid nxt_receiver = self.session.next_receiver(timeout=timeout)
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "<string>", line 6, in next_receiver
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 665, in next_receiver
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid if self._ecwait(lambda: self.incoming, timeout):
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 50, in _ecwait
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid result = self._ewait(lambda: self.closed or predicate(), timeout)
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 571, in _ewait
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid result = self.connection._ewait(lambda: self.error or predicate(), timeout)
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 214, in _ewait
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid self.check_error()
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 207, in check_error
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid raise self.error
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid ConnectionError: connection aborted
2015-01-13 14:41:30.298 1956 TRACE oslo.messaging._drivers.impl_qpid
2015-01-13 14:41:30.392 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 1 seconds
2015-01-13 14:41:31.392 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 2 seconds
2015-01-13 14:41:31.492 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 2 seconds
2015-01-13 14:41:33.492 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 3 seconds
2015-01-13 14:41:33.592 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 3 seconds
2015-01-13 14:41:36.592 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 4 seconds
2015-01-13 14:41:36.692 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 4 seconds
2015-01-13 14:41:49.892 1956 ERROR nova.openstack.common.periodic_task [-] Error during ComputeManager._run_pending_deletes: Timed out waiting for a reply to message ID 64645715b24c465394e9b346c6165fcf
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task Traceback (most recent call last):
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/openstack/common/periodic_task.py", line 182, in run_periodic_tasks
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task task(self, context)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 5708, in _run_pending_deletes
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task context, filters, expected_attrs=attrs)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/objects/base.py", line 110, in wrapper
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task args, kwargs)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/conductor/rpcapi.py", line 425, in object_class_action
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task objver=objver, args=args, kwargs=kwargs)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/client.py", line 150, in call
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task wait_for_reply=True, timeout=timeout)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/transport.py", line 90, in _send
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task timeout=timeout)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 412, in send
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task return self._send(target, ctxt, message, wait_for_reply, timeout)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 403, in _send
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task result = self._waiter.wait(msg_id, timeout)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 280, in wait
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task reply, ending, trylock = self._poll_queue(msg_id, timeout)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 220, in _poll_queue
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task message = self.waiters.get(msg_id, timeout)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 126, in get
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task 'to message ID %s' % msg_id)
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task MessagingTimeout: Timed out waiting for a reply to message ID 64645715b24c465394e9b346c6165fcf
2015-01-13 14:41:49.892 1956 TRACE nova.openstack.common.periodic_task
2015-01-13 14:41:49.897 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Failed to publish message to topic 'conductor': connection aborted
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid Traceback (most recent call last):
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 546, in ensure
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid return method(*args, **kwargs)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 619, in _publisher_send
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid publisher = cls(self.conf, self.session, topic)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 407, in __init__
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid super(TopicPublisher, self).__init__(conf, session, node_name)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 337, in __init__
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid self.reconnect(session)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 341, in reconnect
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid self.sender = session.sender(self.address)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "<string>", line 6, in sender
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 597, in sender
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid sender._ewait(lambda: sender.linked)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 804, in _ewait
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid result = self.session._ewait(lambda: self.error or predicate(), timeout)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 571, in _ewait
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid result = self.connection._ewait(lambda: self.error or predicate(), timeout)
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 214, in _ewait
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid self.check_error()
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 207, in check_error
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid raise self.error
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid ConnectionError: connection aborted
2015-01-13 14:41:49.897 1956 TRACE oslo.messaging._drivers.impl_qpid
2015-01-13 14:42:07.901 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 113] EHOSTUNREACH. Sleeping 1 seconds
2015-01-13 14:42:10.094 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 2 seconds
2015-01-13 14:42:11.694 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 5 seconds
2015-01-13 14:42:11.794 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 5 seconds
2015-01-13 14:42:12.194 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Unable to connect to AMQP server: [Errno 111] ECONNREFUSED. Sleeping 3 seconds
2015-01-13 14:42:15.596 1956 INFO oslo.messaging._drivers.impl_qpid [-] Connected to AMQP server on controller:5672
2015-01-13 14:42:17.097 1956 INFO oslo.messaging._drivers.impl_qpid [-] Connected to AMQP server on controller:5672
2015-01-13 14:42:17.195 1956 INFO oslo.messaging._drivers.impl_qpid [-] Connected to AMQP server on controller:5672
2015-01-13 14:43:15.996 1956 ERROR nova.openstack.common.periodic_task [-] Error during ComputeManager._run_image_cache_manager_pass: Timed out waiting for a reply to message ID cd708c0dbd1c45fda7f9dcfd223eb1d1
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task Traceback (most recent call last):
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/openstack/common/periodic_task.py", line 182, in run_periodic_tasks
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task task(self, context)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 5690, in _run_image_cache_manager_pass
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task filters, expected_attrs=[], use_slave=True)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/objects/base.py", line 110, in wrapper
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task args, kwargs)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/conductor/rpcapi.py", line 425, in object_class_action
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task objver=objver, args=args, kwargs=kwargs)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/client.py", line 150, in call
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task wait_for_reply=True, timeout=timeout)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/transport.py", line 90, in _send
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task timeout=timeout)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 412, in send
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task return self._send(target, ctxt, message, wait_for_reply, timeout)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 403, in _send
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task result = self._waiter.wait(msg_id, timeout)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 280, in wait
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task reply, ending, trylock = self._poll_queue(msg_id, timeout)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 220, in _poll_queue
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task message = self.waiters.get(msg_id, timeout)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 126, in get
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task 'to message ID %s' % msg_id)
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task MessagingTimeout: Timed out waiting for a reply to message ID cd708c0dbd1c45fda7f9dcfd223eb1d1
2015-01-13 14:43:15.996 1956 TRACE nova.openstack.common.periodic_task
2015-01-13 14:43:17.098 1956 WARNING nova.openstack.common.loopingcall [-] task run outlasted interval by 197.510161 sec
2015-01-13 14:44:17.099 1956 ERROR nova.openstack.common.periodic_task [-] Error during ComputeManager._heal_instance_info_cache: Timed out waiting for a reply to message ID 0f0d228f7e5940b7a0cfae2ff2e87c0e
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task Traceback (most recent call last):
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/openstack/common/periodic_task.py", line 182, in run_periodic_tasks
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task task(self, context)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4842, in _heal_instance_info_cache
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task context, self.host, expected_attrs=[], use_slave=True)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/objects/base.py", line 110, in wrapper
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task args, kwargs)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/conductor/rpcapi.py", line 425, in object_class_action
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task objver=objver, args=args, kwargs=kwargs)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/client.py", line 150, in call
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task wait_for_reply=True, timeout=timeout)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/transport.py", line 90, in _send
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task timeout=timeout)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 412, in send
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task return self._send(target, ctxt, message, wait_for_reply, timeout)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 403, in _send
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task result = self._waiter.wait(msg_id, timeout)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 267, in wait
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task reply, ending = self._poll_connection(msg_id, timeout)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 217, in _poll_connection
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task % msg_id)
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task MessagingTimeout: Timed out waiting for a reply to message ID 0f0d228f7e5940b7a0cfae2ff2e87c0e
2015-01-13 14:44:17.099 1956 TRACE nova.openstack.common.periodic_task
2015-01-13 14:45:17.101 1956 WARNING nova.openstack.common.loopingcall [-] task run outlasted interval by 110.002223 sec
2015-01-13 14:45:17.103 1956 ERROR oslo.messaging._drivers.impl_qpid [-] Failed to publish message to topic 'conductor': connection aborted
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid Traceback (most recent call last):
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 546, in ensure
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid return method(*args, **kwargs)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 619, in _publisher_send
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid publisher = cls(self.conf, self.session, topic)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 407, in __init__
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid super(TopicPublisher, self).__init__(conf, session, node_name)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 337, in __init__
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid self.reconnect(session)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 341, in reconnect
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid self.sender = session.sender(self.address)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "<string>", line 6, in sender
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 597, in sender
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid sender._ewait(lambda: sender.linked)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 804, in _ewait
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid result = self.session._ewait(lambda: self.error or predicate(), timeout)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 571, in _ewait
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid result = self.connection._ewait(lambda: self.error or predicate(), timeout)
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 214, in _ewait
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid self.check_error()
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid File "/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 207, in check_error
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid raise self.error
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid ConnectionError: connection aborted
2015-01-13 14:45:17.103 1956 TRACE oslo.messaging._drivers.impl_qpid
2015-01-13 14:45:17.409 1956 INFO oslo.messaging._drivers.impl_qpid [-] Connected to AMQP server on controller:5672
2015-01-13 14:46:17.311 1956 ERROR nova.openstack.common.periodic_task [-] Error during ComputeManager.update_available_resource: Timed out waiting for a reply to message ID 732617b3572640978996a4902d25220f
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task Traceback (most recent call last):
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/openstack/common/periodic_task.py", line 182, in run_periodic_tasks
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task task(self, context)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 5457, in update_available_resource
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task rt.update_available_resource(context)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/openstack/common/lockutils.py", line 249, in inner
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task return f(*args, **kwargs)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/resource_tracker.py", line 315, in update_available_resource
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task context, self.host, self.nodename)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/objects/base.py", line 110, in wrapper
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task args, kwargs)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/conductor/rpcapi.py", line 425, in object_class_action
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task objver=objver, args=args, kwargs=kwargs)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/client.py", line 150, in call
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task wait_for_reply=True, timeout=timeout)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/transport.py", line 90, in _send
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task timeout=timeout)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 412, in send
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task return self._send(target, ctxt, message, wait_for_reply, timeout)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 403, in _send
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task result = self._waiter.wait(msg_id, timeout)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 267, in wait
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task reply, ending = self._poll_connection(msg_id, timeout)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 217, in _poll_connection
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task % msg_id)
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task MessagingTimeout: Timed out waiting for a reply to message ID 732617b3572640978996a4902d25220f
2015-01-13 14:46:17.311 1956 TRACE nova.openstack.common.periodic_task
2015-01-13 14:47:17.312 1956 WARNING nova.openstack.common.loopingcall [-] task run outlasted interval by 110.210403 sec
2015-01-13 14:48:17.313 1956 ERROR nova.openstack.common.periodic_task [-] Error during ComputeManager._cleanup_running_deleted_instances: Timed out waiting for a reply to message ID 1c462d903e654179a60063b6b656261d
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task Traceback (most recent call last):
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/openstack/common/periodic_task.py", line 182, in run_periodic_tasks
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task task(self, context)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 5511, in _cleanup_running_deleted_instances
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task for instance in self._running_deleted_instances(context):
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 5567, in _running_deleted_instances
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task instances = self._get_instances_on_driver(context, filters)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 657, in _get_instances_on_driver
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task context, filters, use_slave=True)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/objects/base.py", line 110, in wrapper
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task args, kwargs)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/nova/conductor/rpcapi.py", line 425, in object_class_action
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task objver=objver, args=args, kwargs=kwargs)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/client.py", line 150, in call
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task wait_for_reply=True, timeout=timeout)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/transport.py", line 90, in _send
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task timeout=timeout)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 412, in send
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task return self._send(target, ctxt, message, wait_for_reply, timeout)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 403, in _send
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task result = self._waiter.wait(msg_id, timeout)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 267, in wait
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task reply, ending = self._poll_connection(msg_id, timeout)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task File "/usr/lib/python2.6/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 217, in _poll_connection
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task % msg_id)
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task MessagingTimeout: Timed out waiting for a reply to message ID 1c462d903e654179a60063b6b656261d
2015-01-13 14:48:17.313 1956 TRACE nova.openstack.common.periodic_task
2015-01-13 14:49:17.316 1956 WARNING nova.openstack.common.loopingcall [-] task run outlasted interval by 110.002466 sec
^C
[root@compute1 ~]#
|