diff --git a/lib/ansible/inventory/group.py b/lib/ansible/inventory/group.py
index dd90c97ea6..0933e1ab5d 100644
--- a/lib/ansible/inventory/group.py
+++ b/lib/ansible/inventory/group.py
@@ -31,6 +31,7 @@ class Group:
         self.depth = 0
         self.name = name
         self.hosts = []
+        self._hosts = set()
         self.vars = {}
         self.child_groups = []
         self.parent_groups = []
@@ -54,11 +55,14 @@ class Group:
         for parent in self.parent_groups:
             parent_groups.append(parent.serialize())
 
+        self._hosts = None
+
         result = dict(
             name=self.name,
             vars=self.vars.copy(),
             parent_groups=parent_groups,
             depth=self.depth,
+            hosts=self.hosts,
         )
 
         return result
@@ -68,6 +72,9 @@ class Group:
         self.name = data.get('name')
         self.vars = data.get('vars', dict())
         self.depth = data.get('depth', 0)
+        self.hosts = data.get('hosts', {})
+
+        self._hosts = set(self.hosts)
 
         parent_groups = data.get('parent_groups', [])
         for parent_data in parent_groups:
@@ -112,17 +119,19 @@ class Group:
             raise AnsibleError("The group named '%s' has a recursive dependency loop." % self.name)
 
     def add_host(self, host):
-        if host in self.hosts:
-            return
-        self.hosts.append(host)
-        host.add_group(self)
-        self.clear_hosts_cache()
+        if host.name not in self._hosts:
+            self.hosts.append(host)
+            self._hosts.add(host.name)
+            host.add_group(self)
+            self.clear_hosts_cache()
 
     def remove_host(self, host):
 
-        self.hosts.remove(host)
-        host.remove_group(self)
-        self.clear_hosts_cache()
+        if host.name in self._hosts:
+            self.hosts.remove(host)
+            self._hosts.remove(host.name)
+            host.remove_group(self)
+            self.clear_hosts_cache()
 
     def set_variable(self, key, value):