MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-01-08 15:09:39 +07:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'traff/helpers'

Browse Source
This commit is contained in:
Dmitry Trofimov
2017-09-25 20:42:34 +02:00
parent 16a3b3c9f1 3dc2aed999
commit 45e9eca92b
12 changed files with 380 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

255
platform/util/src/com/intellij/util/io/TarUtil.java Normal file
View File

@@ -0,0 +1,255 @@
/*
* Copyright 2000-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.io;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.*;
import java.util.Map;
import java.util.Set;
/**
* @author traff
*/
public class TarUtil {
private static final Logger LOG = Logger.getInstance("#com.intellij.util.io.TarUtil");
private TarUtil() {}
@NotNull
public static TarArchiveOutputStream getTarGzOutputStream(File zipFile) throws IOException {
FileOutputStream fos = new FileOutputStream(zipFile);
GzipCompressorOutputStream gcos = new GzipCompressorOutputStream(fos);
TarArchiveOutputStream zip = new TarArchiveOutputStream(gcos);
zip.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
return zip;
}
public interface FileContentProcessor {
FileContentProcessor STANDARD = new FileContentProcessor() {
@Override
public InputStream getContent(File file) throws IOException {
return new FileInputStream(file);
}
};
InputStream getContent(File file) throws IOException;
}
public static boolean addFileToTar(@NotNull TarArchiveOutputStream tos,
@NotNull File file,
@NotNull String relativeName,
@Nullable Set<String> writtenItemRelativePaths,
@Nullable FileFilter fileFilter) throws IOException {
return addFileToTar(tos, file, relativeName, writtenItemRelativePaths, fileFilter, FileContentProcessor.STANDARD);
}
/*
* Adds a new file entry to the TAR output stream.
*/
public static boolean addFileToTar(@NotNull TarArchiveOutputStream tos,
@NotNull File file,
@NotNull String relativeName,
@Nullable Set<String> writtenItemRelativePaths,
@Nullable FileFilter fileFilter,
@NotNull FileContentProcessor contentProcessor) throws IOException {
while (!relativeName.isEmpty() && relativeName.charAt(0) == '/') {
relativeName = relativeName.substring(1);
}
boolean isDir = file.isDirectory();
if (isDir) {
return true;
}
if (fileFilter != null && !FileUtil.isFilePathAcceptable(file, fileFilter)) return false;
if (writtenItemRelativePaths != null && !writtenItemRelativePaths.add(relativeName)) return false;
if (LOG.isDebugEnabled()) {
LOG.debug("Add " + file + " as " + relativeName);
}
long size = file.length();
TarArchiveEntry e = new TarArchiveEntry(relativeName);
e.setModTime(file.lastModified());
e.setSize(size);
tos.putArchiveEntry(e);
InputStream is = contentProcessor.getContent(file);
try {
FileUtil.copy(is, tos);
}
finally {
is.close();
}
tos.closeArchiveEntry();
return true;
}
public static boolean addFileOrDirRecursively(@NotNull TarArchiveOutputStream tarOutputStream,
@Nullable File tarFile,
@NotNull File file,
@NotNull String relativePath,
@Nullable FileFilter fileFilter,
@Nullable Set<String> writtenItemRelativePaths) throws IOException {
if (file.isDirectory()) {
return addDirToTarRecursively(tarOutputStream, tarFile, file, relativePath, fileFilter, writtenItemRelativePaths);
}
addFileToTar(tarOutputStream, file, relativePath, writtenItemRelativePaths, fileFilter);
return true;
}
public static boolean addDirToTarRecursively(@NotNull TarArchiveOutputStream outputStream,
@Nullable File tarFile,
@NotNull File dir,
@NotNull String relativePath,
@Nullable FileFilter fileFilter,
@Nullable Set<String> writtenItemRelativePaths) throws IOException {
if (tarFile != null && FileUtil.isAncestor(dir, tarFile, false)) {
return false;
}
if (!relativePath.isEmpty()) {
addFileToTar(outputStream, dir, relativePath, writtenItemRelativePaths, fileFilter);
}
final File[] children = dir.listFiles();
if (children != null) {
for (File child : children) {
final String childRelativePath = (relativePath.isEmpty() ? "" : relativePath + "/") + child.getName();
addFileOrDirRecursively(outputStream, tarFile, child, childRelativePath, fileFilter, writtenItemRelativePaths);
}
}
return true;
}
public static void extract(@NotNull File file, @NotNull File outputDir, @Nullable FilenameFilter filenameFilter) throws IOException {
extract(file, outputDir, filenameFilter, true);
}
public static void extract(@NotNull File file, @NotNull File outputDir, @Nullable FilenameFilter filenameFilter, boolean overwrite)
throws IOException {
try (FileInputStream fis = new FileInputStream(file);
GzipCompressorInputStream gcis = new GzipCompressorInputStream(fis);
TarArchiveInputStream tis = new TarArchiveInputStream(gcis)) {
extract(tis, outputDir, filenameFilter, overwrite);
}
}
public static void extract(@NotNull final TarArchiveInputStream tis,
@NotNull File outputDir,
@Nullable FilenameFilter filenameFilter) throws IOException {
extract(tis, outputDir, filenameFilter, true);
}
public static void extract(@NotNull final TarArchiveInputStream tis,
@NotNull File outputDir,
@Nullable FilenameFilter filenameFilter,
boolean overwrite) throws IOException {
TarArchiveEntry entry;
while ((entry = tis.getNextTarEntry()) != null) {
final File file = new File(outputDir, entry.getName());
if (filenameFilter == null || filenameFilter.accept(file.getParentFile(), file.getName())) {
extractEntry(entry, tis, outputDir, overwrite);
}
}
}
public static void extractEntry(TarArchiveEntry entry, final InputStream inputStream, File outputDir) throws IOException {
extractEntry(entry, inputStream, outputDir, true);
}
public static void extractEntry(TarArchiveEntry entry, final InputStream inputStream, File outputDir, boolean overwrite)
throws IOException {
final boolean isDirectory = entry.isDirectory();
final String relativeName = entry.getName();
final File file = new File(outputDir, relativeName);
if (file.exists() && !overwrite) return;
FileUtil.createParentDirs(file);
if (isDirectory) {
file.mkdir();
}
else {
if (entry.getSize() > Integer.MAX_VALUE) {
throw new IllegalArgumentException("Tar entries bigger then " + Integer.MAX_VALUE + " aren't supported");
}
int len = (int)entry.getSize();
byte[] content = new byte[len];
int n = 0;
while (n < len) {
int count = inputStream.read(content, n, len - n);
if (count < 0) {
throw new EOFException();
}
n += count;
}
FileUtil.writeToFile(file, content, false);
}
}
/*
* update an existing jar file. Adds/replace files specified in relpathToFile map
*/
public static void update(InputStream in, OutputStream out, Map<String, File> relpathToFile) throws IOException {
try (TarArchiveInputStream tis = new TarArchiveInputStream(in);
TarArchiveOutputStream tos = new TarArchiveOutputStream(out)) {
// put the old entries first, replace if necessary
TarArchiveEntry e;
while ((e = tis.getNextTarEntry()) != null) {
String name = e.getName();
if (!relpathToFile.containsKey(name)) { // copy the old stuff
// do our own compression
TarArchiveEntry e2 = new TarArchiveEntry(name);
//e2.setMethod(e.getMethod());
e2.setModTime(e.getModTime());
//e2.setComment(e.getComment());
//e2.setExtra(e.getExtra());
//if (e.getMethod() == ZipEntry.STORED) {
e2.setSize(e.getSize());
//e2.setCrc(e.getCrc());
//}
tos.putArchiveEntry(e2);
FileUtil.copy(tis, tos);
}
else { // replace with the new files
final File file = relpathToFile.get(name);
//addFile(file, name, tos);
relpathToFile.remove(name);
addFileToTar(tos, file, name, null, null);
}
}
// add the remaining new files
for (final String path : relpathToFile.keySet()) {
File file = relpathToFile.get(path);
addFileToTar(tos, file, path, null, null);
}
}
}
}

0
platform/util/testData/tar/a.txt Normal file
View File

0
platform/util/testData/tar/c.zip Normal file
View File

0
platform/util/testData/tar/dir/b.txt Normal file
View File

0
platform/util/testData/tar/dir/d.zip Normal file
View File

78
platform/util/testSrc/com/intellij/util/io/TarUtilTest.java Normal file
View File

@@ -0,0 +1,78 @@
/*
* Copyright 2000-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.util.io;
import com.google.common.collect.Sets;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.testFramework.PlatformTestUtil;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import org.junit.Test;
import java.io.File;
import java.io.FileFilter;
import java.util.Set;
/**
* @author traff
*/
public class TarUtilTest {
@Test
public void testTarArchiveExtract() throws Exception {
File tempDir = FileUtil.createTempDirectory("tarOutputStream-util-test-", null);
File tarFile = new File(tempDir, "test-tar-util.tar.gz");
TarArchiveOutputStream tarOutputStream = TarUtil.getTarGzOutputStream(tarFile);
Set<String> paths = Sets.newHashSet();
TarUtil.addFileOrDirRecursively(tarOutputStream, null, getTestDataPath(), "",
(FileFilter)pathname -> !pathname.getName().endsWith(".zip"), paths);
tarOutputStream.close();
Assert.assertEquals(2, paths.size());
Assert.assertTrue(paths.stream().noneMatch(f -> f.endsWith(".zip")));
try {
TarUtil.extract(tarFile, tempDir, null);
tarFile.delete();
checkFileStructure(tempDir,
TestFileSystemBuilder.fs()
.file("a.txt")
.dir("dir").file("b.txt"));
}
finally {
tempDir.delete();
}
}
private static void checkFileStructure(@NotNull File parentDir, @NotNull TestFileSystemBuilder expected) {
expected.build().assertDirectoryEqual(parentDir);
}
@NotNull
private static File getTestDataPath() {
File communityDir = new File(PlatformTestUtil.getCommunityPath().replace(File.separatorChar, '/'));
return new File(communityDir, "platform/util/testData/tar");
}
}

3
platform/util/testSrc/util-tests.iml
View File

@@ -18,5 +18,8 @@
<orderEntry type="module" module-name="core-api" scope="TEST" />
<orderEntry type="module" module-name="projectModel-impl" scope="TEST" />
<orderEntry type="module" module-name="jetCheck" scope="TEST" />
<orderEntry type="library" name="commons-compress" level="project" />
<orderEntry type="library" name="Guava" level="project" />
<orderEntry type="module" module-name="testFramework" scope="TEST" />
</component>
</module>

3
platform/util/util.iml
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<module relativePaths="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="true">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8" inherit-compiler-output="true">
<exclude-output />
<annotation-paths>
<root url="file://$MODULE_DIR$/anno" />
@@ -26,6 +26,7 @@
<orderEntry type="library" name="batik" level="project" />
<orderEntry type="library" name="lz4-java" level="project" />
<orderEntry type="library" name="xml-apis-ext" level="project" />
<orderEntry type="library" name="commons-compress" level="project" />
</component>
<component name="copyright">
<Base>

11
python/src/com/jetbrains/python/console/PydevConsoleRunnerImpl.java
View File

@@ -69,7 +69,9 @@ import com.intellij.openapi.vfs.CharsetToolkit;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.wm.ToolWindow;
import com.intellij.psi.PsiFile;
import com.intellij.remote.*;
import com.intellij.remote.CredentialsType;
import com.intellij.remote.RemoteProcess;
import com.intellij.remote.Tunnelable;
import com.intellij.testFramework.LightVirtualFile;
import com.intellij.ui.GuiUtils;
import com.intellij.ui.JBColor;
@@ -530,8 +532,11 @@ public class PydevConsoleRunnerImpl implements PydevConsoleRunner {
public static Couple<Integer> getRemotePortsFromProcess(RemoteProcess process) throws ExecutionException {
Scanner s = new Scanner(process.getInputStream());
return Couple.of(readInt(s, process), readInt(s, process));
try {
return Couple.of(readInt(s, process), readInt(s, process));
} finally {
s.close();
}
}
private static int readInt(Scanner s, Process process) throws ExecutionException {

7
python/src/com/jetbrains/python/packaging/PyRemotePackageManagerImpl.java
View File

@@ -17,6 +17,7 @@ package com.jetbrains.python.packaging;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import com.intellij.execution.ExecutionException;
import com.intellij.execution.process.ProcessOutput;
import com.intellij.openapi.diagnostic.Logger;
@@ -58,7 +59,7 @@ public class PyRemotePackageManagerImpl extends PyPackageManagerImpl {
final SdkAdditionalData sdkData = sdk.getSdkAdditionalData();
if (sdkData instanceof PyRemoteSdkAdditionalDataBase) {
final PyRemoteSdkAdditionalDataBase remoteSdkData = (PyRemoteSdkAdditionalDataBase) sdkData;
final PyRemoteSdkAdditionalDataBase remoteSdkData = (PyRemoteSdkAdditionalDataBase)sdkData;
try {
String helpersPath;
if (CaseCollector.useRemoteCredentials(remoteSdkData)) {
@@ -139,7 +140,9 @@ public class PyRemotePackageManagerImpl extends PyPackageManagerImpl {
workingDir, manager,
remoteSdkAdditionalData,
pathMapper,
askForSudo, true);
askForSudo,
Sets.newHashSet(
helperPath));
}
catch (InterruptedException e) {
throw new ExecutionException(e);

17
python/src/com/jetbrains/python/remote/PythonRemoteInterpreterManager.java
View File

@@ -16,6 +16,7 @@
package com.jetbrains.python.remote;
import com.google.common.base.Function;
import com.google.common.util.concurrent.ListenableFuture;
import com.intellij.execution.ExecutionException;
import com.intellij.execution.configurations.GeneralCommandLine;
import com.intellij.execution.configurations.ParamsGroup;
@@ -112,6 +113,13 @@ public abstract class PythonRemoteInterpreterManager {
public abstract Pair<Supplier<String>, JPanel> createServerBrowserForm(@NotNull final Sdk remoteSdk)
throws ExecutionException, InterruptedException;
public abstract ListenableFuture<?> uploadHelpersAsync(@Nullable Sdk sdk,
@Nullable Project project,
@Nullable Component component,
@NotNull RemoteSdkCredentials credentials, boolean uploadOnSnapshot);
/**
* Short-cut to get {@link PyProjectSynchronizer} for sdk or null if sdk does not have any
*/
@@ -196,4 +204,13 @@ public abstract class PythonRemoteInterpreterManager {
}
public abstract void runVagrant(@NotNull String vagrantFolder, @Nullable String machineName) throws ExecutionException;
/**
* @author traff
*/
public static class PyHelpersNotReadyException extends RuntimeException {
public PyHelpersNotReadyException(Throwable cause) {
super("Python helpers are not copied yet to the remote host. Please wait until remote interpreter initialization finishes.", cause);
}
}
}

12
python/src/com/jetbrains/python/run/PyRemoteProcessStarterManager.java
View File

@@ -27,6 +27,8 @@ import com.jetbrains.python.remote.PythonRemoteInterpreterManager;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Set;
/**
* @author Alexander Koshevoy
*/
@@ -50,6 +52,16 @@ public interface PyRemoteProcessStarterManager {
@NotNull PyRemoteSdkAdditionalDataBase sdkAdditionalData,
@NotNull PyRemotePathMapper pathMapper, boolean askForSudo, boolean checkHelpers) throws ExecutionException, InterruptedException;
default ProcessOutput executeRemoteProcess(@Nullable Project project,
@NotNull String[] command,
@Nullable String workingDir,
@NotNull PythonRemoteInterpreterManager manager,
@NotNull PyRemoteSdkAdditionalDataBase sdkAdditionalData,
@NotNull PyRemotePathMapper pathMapper,
boolean askForSudo,
@NotNull Set<String> checkHelpersPaths) throws ExecutionException, InterruptedException {
return executeRemoteProcess(project, command, workingDir, manager, sdkAdditionalData, pathMapper, askForSudo, !checkHelpersPaths.isEmpty());
}
String getFullInterpreterPath(@NotNull PyRemoteSdkAdditionalDataBase sdkAdditionalData)
throws ExecutionException, InterruptedException;