PY-2813 Do not filter out folders which may be namespace packages

Root folders of namespace packages do not contain __init__.py[co] files. So we traverse every folder in sys.path, which is a valid Python identifier, and take *.py files and setuptools' namespace packages files *-nspkg.pth from all of them except *__pycache__ ones.

python/helpers/generator3.py

@@ -94,7 +94,8 @@ def is_skipped_module(path, f):
 def is_module(d, root):
     return (os.path.exists(os.path.join(root, d, "__init__.py")) or
             os.path.exists(os.path.join(root, d, "__init__.pyc")) or
-            os.path.exists(os.path.join(root, d, "__init__.pyo")))
+            os.path.exists(os.path.join(root, d, "__init__.pyo")) or
+            is_valid_implicit_namespace_package_name(d))
 
 
 def walk_python_path(path):
@@ -166,7 +167,7 @@ def list_sources(paths):
 
             for root, files in walk_python_path(path):
                 for name in files:
-                    if name.endswith('.py'):
+                    if name.endswith('.py') or name.endswith('-nspkg.pth'):
                         file_path = os.path.join(root, name)
                         say("%s\t%s\t%d", os.path.normpath(file_path), path, os.path.getsize(file_path))
         say('END')

python/helpers/pycharm_generator_utils/util_methods.py

@@ -704,3 +704,28 @@ def build_output_name(dirname, qualified_name):
         os.makedirs(dirname)
 
     return fname
+
+
+def is_valid_implicit_namespace_package_name(s):
+    """
+    Checks whether provided string could represent implicit namespace package name.
+    :param s: string to check
+    :return: True if provided string could represent implicit namespace package name and False otherwise
+    """
+    return isidentifier(s) and not keyword.iskeyword(s)
+
+
+def isidentifier(s):
+    """
+    Checks whether provided string complies Python identifier syntax requirements.
+    :param s: string to check
+    :return: True if provided string comply Python identifier syntax requirements and False otherwise
+    """
+    if version[0] >= 3:
+        return s.isidentifier()
+    else:
+        # quick test on provided string to comply major Python identifier syntax requirements
+        return (s and
+                not s[:1].isdigit() and
+                "-" not in s and
+                " " not in s)