Remove unused skeletons and move stubs whitelist to syncing script

This commit is contained in:
Semyon Proshev
2018-06-04 19:22:58 +03:00
parent cb981a0ef5
commit 5eb1239228
8 changed files with 84 additions and 6367 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,56 +0,0 @@
"""Skeleton for 'collections' stdlib module."""
import sys
import collections
class Iterable(object):
def __init__(self):
"""
:rtype: collections.Iterable[T]
"""
pass
def __iter__(self):
"""
:rtype: collections.Iterator[T]
"""
class Iterator(collections.Iterable):
def __init__(self):
"""
:rtype: collections.Iterator[T]
"""
pass
if sys.version_info >= (3, 0):
def __next__(self):
"""
:rtype: T
"""
pass
if sys.version_info < (3, 0):
def next(self):
"""
:rtype: T
"""
pass
class defaultdict(dict):
def __init__(self, default_factory=None, **kwargs):
"""
:type default_factory: () -> V
:rtype: defaultdict[Any, V]
"""
pass
def __missing__(self, key):
"""
:type key: Any
:rtype: V
"""
pass
-625
View File
@@ -1,625 +0,0 @@
"""Skeleton for 'datetime' stdlib module."""
import sys
import datetime as _datetime
from time import struct_time
class timedelta(object):
"""A timedelta object represents a duration, the difference between two
dates or times."""
def __init__(self, days=0, seconds=0, microseconds=0, milliseconds=0,
minutes=0, hours=0, weeks=0):
"""Create a timedelta object.
:type days: numbers.Real
:type seconds: numbers.Real
:type microseconds: numbers.Real
:type milliseconds: numbers.Real
:type minutes: numbers.Real
:type hours: numbers.Real
:type weeks: numbers.Real
"""
self.days = 0
self.seconds = 0
self.microseconds = 0
def __add__(self, other):
"""Add timedelta, date or datetime.
:type other: T <= _datetime.timedelta | _datetime.date | _datetime.datetime
:rtype: T
"""
pass
def __radd__(self, other):
"""Add timedelta, date or datetime.
:type other: T <= _datetime.timedelta | _datetime.date | _datetime.datetime
:rtype: T
"""
pass
def __sub__(self, other):
"""Subtract timedelta, date or datetime.
:type other: _datetime.timedelta | _datetime.date | _datetime.datetime
:rtype: _datetime.timedelta | _datetime.date | _datetime.datetime
"""
pass
def __rsub__(self, other):
"""Subtract timedelta, date or datetime.
:type other: _datetime.timedelta | _datetime.date | _datetime.datetime
:rtype: _datetime.timedelta | _datetime.date | _datetime.datetime
"""
pass
def __mul__(self, other):
"""Multiply by an integer.
:type other: numbers.Integral
:rtype: _datetime.timedelta
"""
return _datetime.timedelta()
def __rmul__(self, other):
"""Multiply by an integer.
:type other: numbers.Integral
:rtype: _datetime.timedelta
"""
return _datetime.timedelta()
def __floordiv__(self, other):
"""Divide by an integer or a timedelta.
:type other: numbers.Integral | _datetime.timedelta
:rtype: _datetime.timedelta | int
"""
pass
def __div__(self, other):
"""Divide by an integer.
:type other: numbers.Integral
:rtype: _datetime.timedelta
"""
pass
def __truediv__(self, other):
"""Divide by a float or a timedelta.
:type other: numbers.Real | _datetime.timedelta
:rtype: _datetime.timedelta | float
"""
pass
if sys.version_info >= (2, 7):
def total_seconds(self):
"""Return the total number of seconds contained in the duration.
:rtype: int
"""
return 0
min = _datetime.timedelta()
max = _datetime.timedelta()
resoultion = _datetime.timedelta()
class date(object):
"""An idealized naive date, assuming the current Gregorian calendar always
was, and always will be, in effect."""
def __init__(self, year, month, day):
"""Create a date object.
:type year: numbers.Integral
:type month: numbers.Integral
:type day: numbers.Integral
"""
self.year = year
self.month = month
self.day = day
@classmethod
def today(cls):
"""Return the current local date.
:rtype: _datetime.date
"""
return _datetime.date(0, 0, 0)
@classmethod
def fromtimestamp(cls, timestamp):
"""Return the local date corresponding to the POSIX timestamp, such as
is returned by time.time().
:type timestamp: numbers.Real
:rtype: _datetime.date
"""
return _datetime.date(0, 0, 0)
@classmethod
def fromordinal(cls, ordinal):
"""Return the date corresponding to the proleptic Gregorian ordinal,
where January 1 of year 1 has ordinal 1.
:type ordinal: numbers.Integral
:rtype: _datetime.date
"""
return _datetime.date(0, 0, 0)
def __add__(self, other):
"""Add timedelta.
:type other: _datetime.timedelta
:rtype: _datetime.date
"""
return _datetime.date(0, 0, 0)
def __radd__(self, other):
"""Add timedelta.
:type other: _datetime.timedelta
:rtype: _datetime.date
"""
return _datetime.date(0, 0, 0)
def __sub__(self, other):
"""Subtract date or timedelta.
:type other: _datetime.date | _datetime.timedelta
:rtype: _datetime.timedelta | _datetime.date
"""
pass
def __rsub__(self, other):
"""Subtract date.
:type other: _datetime.date
:rtype: _datetime.timedelta
"""
return _datetime.timedelta()
def replace(self, year=None, month=None, day=None):
"""Return a date with the same value, except for those parameters given
new values by whichever keyword arguments are specified.
:type year: numbers.Integral
:type month: numbers.Integral
:type day: numbers.Integral
:rtype: _datetime.date
"""
return _datetime.date(0, 0, 0)
def timetuple(self):
"""Return a time.struct_time such as returned by time.localtime().
:rtype: struct_time
"""
return struct_time()
def toordinal(self):
"""Return the proleptic Gregorian ordinal of the date, where January 1
of year 1 has ordinal 1.
:rtype: int
"""
return 0
def weekday(self):
"""Return the day of the week as an integer, where Monday is 0 and
Sunday is 6.
:rtype: int
"""
return 0
def isoweekday(self):
"""Return the day of the week as an integer, where Monday is 1 and
Sunday is 7.
:rtype: int
"""
return 0
def isocalendar(self):
"""Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
:rtype: (int, int, int)
"""
return (0, 0, 0)
def isoformat(self):
"""Return a string representing the date in ISO 8601 format,
'YYYY-MM-DD'.
:rtype: string
"""
return str()
def ctime(self):
"""Return a string representing the date.
:rtype: string
"""
return str()
def strftime(self, format):
"""Return a string representing the date, controlled by an explicit
format string.
:type format: string
:rtype: string
"""
return str()
min = _datetime.date(0, 0, 0)
max = _datetime.date(0, 0, 0)
resoultion = _datetime.timedelta()
class datetime(object):
"""A datetime object is a single object containing all the information from
a date object and a time object."""
def __init__(self, year, month, day, hour=0, minute=0, second=0,
microsecond=0, tzinfo=None):
"""Create a datetime object.
:type year: numbers.Integral
:type month: numbers.Integral
:type day: numbers.Integral
:type hour: numbers.Integral
:type minute: numbers.Integral
:type second: numbers.Integral
:type microsecond: numbers.Integral
:type tzinfo: _datetime.tzinfo | None
"""
self.year = year
self.month = month
self.day = day
self.hour = hour
self.minute = minute
self.second = second
self.microsecond = microsecond
self.tzinfo = tzinfo
@classmethod
def today(cls):
"""Return the current local datetime, with tzinfo None.
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
@classmethod
def now(cls, tz=None):
"""Return the current local date and time.
:type tz: _datetime.tzinfo | None
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
@classmethod
def utcnow(cls):
"""Return the current UTC date and time, with tzinfo None.
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
@classmethod
def fromtimestamp(cls, timestamp, tz=None):
"""Return the local date and time corresponding to the POSIX timestamp,
such as is returned by time.time().
:type timestamp: numbers.Real
:type tz: _datetime.tzinfo | None
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
@classmethod
def utcfromtimestamp(cls, timestamp):
"""Return the UTC datetime corresponding to the POSIX timestamp, with
tzinfo None.
:type timestamp: numbers.Real
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
@classmethod
def fromordinal(cls, ordinal):
"""Return the datetime corresponding to the proleptic Gregorian
ordinal.
:type ordinal: numbers.Integral
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
@classmethod
def combine(cls, date, time):
"""Return a new datetime object whose date components are equal to the
given date object's, and whose time components and tzinfo attributes
are equal to the given time object's.
:type date: _datetime.date
:type time: _datetime.time
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
@classmethod
def strptime(cls, date_string, format):
"""Return a datetime corresponding to date_string, parsed according to
format.
:type date_string: string
:type format: string
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
def __add__(self, other):
"""Add timedelta.
:type other: _datetime.timedelta
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
def __radd__(self, other):
"""Add timedelta.
:type other: _datetime.timedelta
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
def __sub__(self, other):
"""Subtract timedelta or datetime.
:type other: _datetime.timedelta | _datetime.datetime
:rtype: _datetime.datetime | _datetime.timedelta
"""
pass
def __rsub__(self, other):
"""Subtract datetime.
:type other: _datetime.datetime
:rtype: _datetime.timedelta
"""
return _datetime.timedelta()
def date(self):
"""Return date object with same year, month and day.
:rtype: _datetime.date
"""
return _datetime.date(0, 0, 0)
def time(self):
"""Return time object with same hour, minute, second and microsecond.
:rtype: _datetime.time
"""
return _datetime.time()
def timetz(self):
"""Return time object with same hour, minute, second, microsecond, and
tzinfo attributes.
:rtype: _datetime.time
"""
return _datetime.time()
def replace(self, year=None, month=None, day=None, hour=None, minute=None,
second=None, microsecond=None, tzinfo=None):
"""Return a datetime with the same attributes, except for those
attributes given new values by whichever keyword arguments are
specified.
:type year: numbers.Integral
:type month: numbers.Integral
:type day: numbers.Integral
:type hour: numbers.Integral
:type minute: numbers.Integral
:type second: numbers.Integral
:type microsecond: numbers.Integral
:type tzinfo: _datetime.tzinfo | None
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
def astimezone(self, tz):
"""Return a datetime object with new tzinfo attribute tz, adjusting the
date and time data so the result is the same UTC time as self, but in
tz's local time.
:type tz: _datetime.tzinfo
:rtype: _datetime.datetime
"""
return _datetime.datetime(0, 0, 0)
def utcoffset(self):
"""If tzinfo is None, returns None, else returns
self.tzinfo.utcoffset(self).
:rtype: _datetime.timedelta | None
"""
return _datetime.timedelta()
def dst(self):
"""If tzinfo is None, returns None, else returns self.tzinfo.dst(self).
:rtype: _datetime.timedelta | None
"""
return _datetime.timedelta()
def tzname(self):
"""If tzinfo is None, returns None, else returns
self.tzinfo.tzname(self).
:rtype: string | None
"""
return str()
def timetuple(self):
"""Return a time.struct_time such as returned by time.localtime().
:rtype: struct_time
"""
return struct_time()
def utctimetuple(self):
"""If datetime instance d is naive, this is the same as d.timetuple()
except that tm_isdst is forced to 0 regardless of what d.dst() returns.
:rtype: struct_time
"""
return struct_time()
def toordinal(self):
"""Return the proleptic Gregorian ordinal of the date.
:rtype: int
"""
return 0
def weekday(self):
"""Return the day of the week as an integer, where Monday is 0 and
Sunday is 6.
:rtype: int
"""
return 0
def isoweekday(self):
"""Return the day of the week as an integer, where Monday is 1 and
Sunday is 7.
:rtype: int
"""
return 0
def isocalendar(self):
"""Return a 3-tuple, (ISO year, ISO week number, ISO weekday).
:rtype: (int, int, int)
"""
return (0, 0, 0)
def isoformat(self, sep='T'):
"""Return a string representing the date and time in ISO 8601 format.
:type sep: string
:rtype: string
"""
return str()
def ctime(self):
"""Return a string representing the date and time.
:rtype: string
"""
return str()
def strftime(self, format):
"""Return a string representing the date and time, controlled by an
explicit format string.
:type format: string
:rtype: string
"""
return str()
min = _datetime.datetime(0, 0, 0)
max = _datetime.datetime(0, 0, 0)
resoultion = _datetime.timedelta()
class time(object):
"""A time object represents a (local) time of day, independent of any
particular day, and subject to adjustment via a tzinfo object."""
def __init__(self, hour=0, minute=0, second=0, microsecond=0, tzinfo=None):
"""Create a time object.
:type hour: numbers.Integral
:type minute: numbers.Integral
:type second: numbers.Integral
:type microsecond: numbers.Integral
:type tzinfo: _datetime.tzinfo | None
"""
self.hour = hour
self.minute = minute
self.second = second
self.microsecond = microsecond
sefl.tzinfo = tzinfo
def replace(self, hour=None, minute=None, second=None, microsecond=None,
tzinfo=None):
"""Return a time with the same value, except for those attributes given
new values by whichever keyword arguments are specified.
:type hour: numbers.Integral
:type minute: numbers.Integral
:type second: numbers.Integral
:type microsecond: numbers.Integral
:type tzinfo: _datetime.tzinfo | None
:rtype: _datetime.time
"""
return _datetime.time()
def isoformat(self):
"""Return a string representing the time in ISO 8601 format.
:rtype: string
"""
return str()
def strftime(self, format):
"""Return a string representing the time, controlled by an explicit
format string.
:type format: string
:rtype: string
"""
return str()
def utcoffset(self):
"""If tzinfo is None, returns None, else returns
self.tzinfo.utcoffset(self).
:rtype: _datetime.timedelta | None
"""
return _datetime.timedelta()
def dst(self):
"""If tzinfo is None, returns None, else returns self.tzinfo.dst(self).
:rtype: _datetime.timedelta | None
"""
return _datetime.timedelta()
def tzname(self):
"""If tzinfo is None, returns None, else returns
self.tzinfo.tzname(self).
:rtype: string | None
"""
return str()
min = _datetime.time()
max = _datetime.time()
resoultion = _datetime.timedelta()
@@ -23,7 +23,6 @@ import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.util.QualifiedName
import com.jetbrains.python.PythonHelpersLocator
import com.jetbrains.python.codeInsight.typing.PyTypingTypeProvider.TYPING
import com.jetbrains.python.packaging.PyPIPackageUtil
import com.jetbrains.python.packaging.PyPackageManagers
import com.jetbrains.python.packaging.PyPackageUtil
@@ -39,24 +38,13 @@ import java.io.File
* @author vlan
*/
object PyTypeShed {
private val ONLY_SUPPORTED_PY2_MINOR = 7
private const val ONLY_SUPPORTED_PY2_MINOR = 7
private val SUPPORTED_PY3_MINORS = 2..7
val WHITE_LIST: Set<String> = setOf(TYPING, "six", "__builtin__", "builtins", "exceptions", "types", "datetime", "functools", "shutil",
"re", "time", "argparse", "uuid", "threading", "signal", "collections", "subprocess", "math", "queue",
"socket", "sqlite3", "attr")
private val BLACK_LIST = setOf<String>()
/**
* Returns true if we allow to search typeshed for a stub for [name].
*/
fun maySearchForStubInRoot(name: QualifiedName, root: VirtualFile, sdk : Sdk): Boolean {
val topLevelPackage = name.firstComponent ?: return false
if (topLevelPackage in BLACK_LIST) {
return false
}
if (topLevelPackage !in WHITE_LIST) {
return false
}
if (isInStandardLibrary(root)) {
return true
}
@@ -64,6 +52,7 @@ object PyTypeShed {
if (ApplicationManager.getApplication().isUnitTestMode) {
return true
}
val topLevelPackage = name.firstComponent ?: return false
val pyPIPackages = PyPIPackageUtil.PACKAGES_TOPLEVEL[topLevelPackage] ?: emptyList()
val packages = PyPackageManagers.getInstance().forSdk(sdk).packages ?: return true
return PyPackageUtil.findPackage(packages, topLevelPackage) != null ||
@@ -34,7 +34,6 @@ import com.intellij.util.containers.ContainerUtil;
import com.jetbrains.python.PythonHelpersLocator;
import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil;
import com.jetbrains.python.codeInsight.typing.PyTypeShed;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.resolve.*;
import com.jetbrains.python.psi.types.PyClassLikeType;
@@ -61,27 +60,17 @@ public class PyUserSkeletonsUtil {
"asyncio",
"multiprocessing",
"os",
"__builtin__.py",
"_csv.py",
"builtins.py",
"collections.py",
"copy.py",
"cStringIO.py",
"datetime.py",
"decimal.py",
"functools.py",
"io.py",
"itertools.py",
"logging.py",
"math.py",
"pathlib.py",
"pickle.py",
"re.py",
"shutil.py",
"sqlite3.py",
"StringIO.py",
"struct.py",
"subprocess.py",
"sys.py"
);
@@ -237,10 +226,6 @@ public class PyUserSkeletonsUtil {
if (moduleVirtualFile != null) {
String moduleName = QualifiedNameFinder.findShortestImportableName(file, moduleVirtualFile);
if (moduleName != null) {
// TODO: Delete user-skeletons altogether, meanwhile disabled user-skeletons for modules already covered by PyTypeShed
if (PyTypeShed.INSTANCE.getWHITE_LIST().contains(moduleName)) {
return null;
}
final QualifiedName qName = QualifiedName.fromDottedString(moduleName);
final QualifiedName restored = QualifiedNameFinder.canonizeQualifiedName(qName, null);
if (restored != null) {
@@ -1,78 +0,0 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.tools
import com.intellij.openapi.util.io.FileUtil
import com.intellij.util.io.createDirectories
import com.intellij.util.io.delete
import com.intellij.util.io.exists
import com.jetbrains.python.codeInsight.typing.PyTypeShed
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
fun main(args: Array<String>) {
val repo = Paths.get("../typeshed")
val bundled = Paths.get("./community/python/helpers/typeshed")
println("Repo: ${repo.abs()}")
println("Bundled: ${bundled.abs()}")
sync(repo, bundled)
clean(topLevelPackages(bundled), PyTypeShed.WHITE_LIST)
}
private fun sync(repo: Path, bundled: Path) {
if (!repo.exists()) throw IllegalArgumentException("Not found: ${repo.abs()}")
if (bundled.exists()) {
bundled.delete()
println("Removed: ${bundled.abs()}")
}
bundled.createDirectories()
if (!bundled.exists()) throw IllegalStateException("Not found: ${bundled.abs()}")
val whiteList = setOf("stdlib",
"tests",
"third_party",
".flake8",
".gitignore",
".travis.yml",
"CONTRIBUTING.md",
"README.md",
"LICENSE",
"requirements-tests-py2.txt",
"requirements-tests-py3.txt")
Files
.newDirectoryStream(repo)
.forEach {
if (it.name() in whiteList) {
val target = bundled.resolve(it.fileName)
it.toFile().copyRecursively(target.toFile())
println("Copied: ${it.abs()} to ${target.abs()}")
}
else {
println("Skipped: ${it.abs()}")
}
}
}
private fun topLevelPackages(typeshed: Path): List<Path> {
return sequenceOf(typeshed)
.flatMap { sequenceOf(it.resolve("stdlib"), it.resolve("third_party")) }
.flatMap { Files.newDirectoryStream(it).asSequence() }
.flatMap { Files.newDirectoryStream(it).asSequence() }
.toList()
}
private fun clean(topLevelPackages: List<Path>, whiteList: Set<String>) {
topLevelPackages
.asSequence()
.filter { FileUtil.getNameWithoutExtension(it.name()) !in whiteList }
.forEach { it.delete() }
}
private fun Path.abs() = toAbsolutePath()
private fun Path.name() = fileName.toString()
@@ -0,0 +1,82 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.tools
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
/**
* This script was implemented to sync local copy of `typeshed` with bundled `typeshed`.
*
* As a result it leaves top-level modules and packages that are listed in `whiteList`.
* It allows us to reduce the size of bundled `typeshed` and do not run indexing and other analyzing processes on disabled stubs.
*/
val repo = Paths.get("../../../../../../../../../typeshed").abs().normalize()
val bundled = Paths.get("../../../../../../../../community/python/helpers/typeshed").abs().normalize()
println("Repo: ${repo.abs()}")
println("Bundled: ${bundled.abs()}")
sync(repo, bundled)
val whiteList = setOf("typing", "six", "__builtin__", "builtins", "exceptions", "types", "datetime", "functools", "shutil", "re", "time",
"argparse", "uuid", "threading", "signal", "collections", "subprocess", "math", "queue", "socket", "sqlite3", "attr")
clean(topLevelPackages(bundled), whiteList)
fun sync(repo: Path, bundled: Path) {
if (!Files.exists(repo)) throw IllegalArgumentException("Not found: ${repo.abs()}")
if (Files.exists(bundled)) {
bundled.deleteRecursively()
println("Removed: ${bundled.abs()}")
}
val whiteList = setOf("stdlib",
"tests",
"third_party",
".flake8",
".gitignore",
".travis.yml",
"CONTRIBUTING.md",
"README.md",
"LICENSE",
"requirements-tests-py2.txt",
"requirements-tests-py3.txt")
Files
.newDirectoryStream(repo)
.forEach {
if (it.name() in whiteList) {
val target = bundled.resolve(it.fileName)
it.copyRecursively(target)
println("Copied: ${it.abs()} to ${target.abs()}")
}
else {
println("Skipped: ${it.abs()}")
}
}
}
fun topLevelPackages(typeshed: Path): List<Path> {
return sequenceOf(typeshed)
.flatMap { sequenceOf(it.resolve("stdlib"), it.resolve("third_party")) }
.flatMap { Files.newDirectoryStream(it).asSequence() }
.flatMap { Files.newDirectoryStream(it).asSequence() }
.toList()
}
fun clean(topLevelPackages: List<Path>, whiteList: Set<String>) {
topLevelPackages
.asSequence()
.filter { it.nameWithoutExtension() !in whiteList }
.forEach { it.deleteRecursively() }
}
fun Path.abs() = toAbsolutePath()
fun Path.deleteRecursively() = toFile().deleteRecursively()
fun Path.copyRecursively(target: Path) = toFile().copyRecursively(target.toFile())
fun Path.name() = toFile().name
fun Path.nameWithoutExtension() = toFile().nameWithoutExtension