From 6f21733a53325fbb30d0cccaf46fb492ddc3f589 Mon Sep 17 00:00:00 2001 From: Alexey Kudravtsev Date: Sun, 20 Aug 2017 16:34:16 +0300 Subject: [PATCH] IDEA-177424 Corrupting file encoding upon change For files (pre)detected as text, perform charset detection using the entire file content (instead of 8K buffer) to avoid misdetecting files with UTF-8-specific byte sequences far into the text as US-ASCII. Balk out early if detected binary from 8K buffer. --- .../codeInsight/daemon/LossyEncodingTest.java | 37 +- .../openapi/fileEditor/impl/LoadTextUtil.java | 82 +- .../fileTypes/impl/FileTypeManagerImpl.java | 177 +- .../testData/vfs/encoding/BIGCHANGES | 3186 +++++++++++++++++ .../vfs/encoding/FileEncodingTest.java | 9 + 5 files changed, 3343 insertions(+), 148 deletions(-) create mode 100644 platform/platform-tests/testData/vfs/encoding/BIGCHANGES diff --git a/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/LossyEncodingTest.java b/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/LossyEncodingTest.java index 4fe28baf13d1..8391fff4ae25 100644 --- a/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/LossyEncodingTest.java +++ b/java/java-tests/testSrc/com/intellij/java/codeInsight/daemon/LossyEncodingTest.java @@ -21,15 +21,14 @@ import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerEx; import com.intellij.codeInsight.daemon.impl.HighlightInfo; import com.intellij.codeInspection.LocalInspectionTool; import com.intellij.codeInspection.LossyEncodingInspection; -import com.intellij.openapi.command.WriteCommandAction; import com.intellij.openapi.editor.Document; import com.intellij.openapi.fileEditor.FileDocumentManager; -import com.intellij.openapi.fileTypes.FileTypes; import com.intellij.openapi.fileTypes.StdFileTypes; import com.intellij.openapi.vfs.CharsetToolkit; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.encoding.EncodingManager; import com.intellij.openapi.vfs.encoding.EncodingProjectManager; +import com.intellij.util.ObjectUtils; import com.intellij.util.ui.UIUtil; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; @@ -127,37 +126,11 @@ public class LossyEncodingTest extends DaemonAnalyzerTestCase { doDoTest(true, false); } - public void testDetectWrongEncoding0() throws Exception { - String threeNotoriousRussianLetters = "\u0416\u041e\u041f"; - configureByText(FileTypes.PLAIN_TEXT, threeNotoriousRussianLetters); - VirtualFile virtualFile = getFile().getVirtualFile(); - final Document document = FileDocumentManager.getInstance().getDocument(virtualFile); - WriteCommandAction.runWriteCommandAction(getProject(), () -> { - document.insertString(0, " "); - document.deleteString(0, 1); - }); - - - assertTrue(FileDocumentManager.getInstance().isDocumentUnsaved(document)); - assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset()); - Charset WINDOWS_1251 = Charset.forName("windows-1251"); - virtualFile.setCharset(WINDOWS_1251); - FileDocumentManager.getInstance().saveAllDocuments(); // save in wrong encoding - assertEquals(WINDOWS_1251, virtualFile.getCharset()); - assertEquals(threeNotoriousRussianLetters, new String(virtualFile.contentsToByteArray(), WINDOWS_1251)); - virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET); - - doHighlighting(); - List infos = DaemonCodeAnalyzerEx.getInstanceEx(getProject()).getFileLevelHighlights(getProject(), getFile()); - HighlightInfo info = assertOneElement(infos); - assertEquals("File was loaded in the wrong encoding: 'UTF-8'", info.getDescription()); - } - public void testDetectWrongEncoding() { - VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + "Win1251.txt"); + VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/Win1251.txt"); virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET); configureByExistingFile(virtualFile); - final Document document = FileDocumentManager.getInstance().getDocument(virtualFile); + Document document = ObjectUtils.notNull(FileDocumentManager.getInstance().getDocument(virtualFile)); assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document)); assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset()); @@ -172,7 +145,7 @@ public class LossyEncodingTest extends DaemonAnalyzerTestCase { VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + "surrogate.txt"); virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET); configureByExistingFile(virtualFile); - final Document document = FileDocumentManager.getInstance().getDocument(virtualFile); + final Document document = ObjectUtils.notNull(FileDocumentManager.getInstance().getDocument(virtualFile)); assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document)); assertEquals(CharsetToolkit.UTF8_CHARSET, virtualFile.getCharset()); @@ -184,7 +157,7 @@ public class LossyEncodingTest extends DaemonAnalyzerTestCase { VirtualFile virtualFile = getVirtualFile(BASE_PATH + "/" + getTestName(false) + ".txt"); configureByExistingFile(virtualFile); FileDocumentManager.getInstance().saveAllDocuments(); - final Document document = FileDocumentManager.getInstance().getDocument(virtualFile); + final Document document = ObjectUtils.notNull(FileDocumentManager.getInstance().getDocument(virtualFile)); assertFalse(FileDocumentManager.getInstance().isDocumentUnsaved(document)); doHighlighting(); List infos = DaemonCodeAnalyzerEx.getInstanceEx(getProject()).getFileLevelHighlights(getProject(), getFile()); diff --git a/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java b/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java index 94b5fec2d631..91c17ac103e6 100644 --- a/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java +++ b/platform/core-impl/src/com/intellij/openapi/fileEditor/impl/LoadTextUtil.java @@ -47,7 +47,6 @@ import java.nio.CharBuffer; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CharsetEncoder; -import java.util.function.Function; public final class LoadTextUtil { private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.fileEditor.impl.LoadTextUtil"); @@ -194,42 +193,32 @@ public final class LoadTextUtil { } } + // guess from file type or content @NotNull - private static Trinity detectCharset(@NotNull VirtualFile virtualFile, - @NotNull byte[] content, - int startOffset, int endOffset, - @NotNull FileType fileType, - @NotNull Function computeCharsetIfNotDetected) { - Charset charset = null; + private static Trinity detectHardCharset(@NotNull VirtualFile virtualFile, + @NotNull byte[] content, + int startOffset, int endOffset, + @NotNull FileType fileType) { + Charset hardCodedCharset; String charsetName = fileType.getCharset(virtualFile, content); Trinity guessed = guessFromContent(virtualFile, content, startOffset, endOffset); if (charsetName == null) { - Charset hardCodedCharset = guessed == null ? null : guessed.first; - - if (hardCodedCharset != null) { - charset = hardCodedCharset; - } + hardCodedCharset = guessed == null ? null : guessed.first; } else { - charset = CharsetToolkit.forName(charsetName); + hardCodedCharset = CharsetToolkit.forName(charsetName); } - if (charset == null) { - charset = computeCharsetIfNotDetected.apply(virtualFile); + if (hardCodedCharset == null && guessed != null && guessed.second != null && guessed.second == CharsetToolkit.GuessedEncoding.VALID_UTF8) { + return Trinity.create(CharsetToolkit.UTF8_CHARSET, guessed.getSecond(), guessed.getThird()); } - if (charset == null && guessed != null && guessed.second != null) { - if (guessed.second == CharsetToolkit.GuessedEncoding.VALID_UTF8) return Trinity.create(CharsetToolkit.UTF8_CHARSET, guessed.getSecond(),guessed.getThird()); - if (guessed.second == CharsetToolkit.GuessedEncoding.SEVEN_BIT) return Trinity.create(CharsetToolkit.US_ASCII_CHARSET, guessed.getSecond(),guessed.getThird()); - } - return Trinity.create(charset, guessed == null ? null : guessed.getSecond(), guessed == null ? null : guessed.getThird()); + return Trinity.create(hardCodedCharset, guessed == null ? null : guessed.getSecond(), guessed == null ? null : guessed.getThird()); } @NotNull public static Charset detectCharsetAndSetBOM(@NotNull VirtualFile virtualFile, @NotNull byte[] content, @NotNull FileType fileType) { - Charset internalCharset = doDetectCharsetAndSetBOM(virtualFile, content, 0,content.length, true, fileType, - virtualFile.isCharsetSet() ? virtualFile.getCharset() : null, - LoadTextUtil::getDefaultCharsetFromEncodingManager).getFirst(); + Charset internalCharset = detectInternalCharsetAndSetBOM(virtualFile, content, 0, content.length, true, fileType).getFirst(); return internalCharset instanceof SevenBitCharset ? ((SevenBitCharset)internalCharset).myBaseCharset : internalCharset; } @@ -247,32 +236,31 @@ public final class LoadTextUtil { } @NotNull - private static Trinity doDetectCharsetAndSetBOM(@NotNull VirtualFile virtualFile, - @NotNull byte[] content, - int startOffset, int endOffset, - boolean saveBOM, - @NotNull FileType fileType, - @Nullable Charset initialCharset, - @NotNull Function computeCharsetIfNotDetected) { - Trinity - info = detectCharset(virtualFile, content, startOffset, endOffset, fileType, computeCharsetIfNotDetected); - Charset detectedCharset = info.getFirst(); + private static Trinity + detectInternalCharsetAndSetBOM(@NotNull VirtualFile file, + @NotNull byte[] content, + int startOffset, int endOffset, + boolean saveBOM, + @NotNull FileType fileType) { + Trinity info = detectHardCharset(file, content, startOffset, endOffset, fileType); + + Charset detectedHardCharset = info.getFirst(); CharsetToolkit.GuessedEncoding guessed = info.getSecond(); byte[] bom = info.getThird(); - Charset charset = initialCharset != null ? initialCharset : detectedCharset; - // can be overridden by BOM - Charset fromBOM = bom == null ? null : detectedCharset; - // but should not override native_to_ascii wrapped utf-XXX - if (fromBOM != null && (!charset.name().startsWith("NATIVE_TO_ASCII_") || !charset.name().endsWith(fromBOM.name()))) { - charset = fromBOM; + Charset charset; + if (detectedHardCharset == null) { + charset = file.isCharsetSet() ? file.getCharset() : getDefaultCharsetFromEncodingManager(file); + } + else { + charset = detectedHardCharset; } if (saveBOM && bom != null && bom.length != 0) { - virtualFile.setBOM(bom); - setCharsetWasDetectedFromBytes(virtualFile, AUTO_DETECTED_FROM_BOM); + file.setBOM(bom); + setCharsetWasDetectedFromBytes(file, AUTO_DETECTED_FROM_BOM); } - virtualFile.setCharset(charset); + file.setCharset(charset); Charset result = charset; // optimisation @@ -306,7 +294,7 @@ public final class LoadTextUtil { info = null; } else { - Charset defaultCharset = ObjectUtils.notNull(EncodingManager.getInstance().getEncoding(virtualFile, true), CharsetToolkit.getDefaultSystemCharset()); + Charset defaultCharset = getDefaultCharsetFromEncodingManager(virtualFile); info = guessFromBytes(content, startOffset, endOffset, defaultCharset); byte[] bom = info.getThird(); CharsetToolkit.GuessedEncoding guessed = info.getSecond(); @@ -580,9 +568,7 @@ public final class LoadTextUtil { boolean saveDetectedSeparators, boolean saveBOM) { Trinity - info = doDetectCharsetAndSetBOM(virtualFile, bytes, 0, bytes.length, saveBOM, virtualFile.getFileType(), - virtualFile.isCharsetSet() ? virtualFile.getCharset() : null, - LoadTextUtil::getDefaultCharsetFromEncodingManager); + info = detectInternalCharsetAndSetBOM(virtualFile, bytes, 0, bytes.length, saveBOM, virtualFile.getFileType()); Charset internalCharset = info.getFirst(); byte[] bom = info.getThird(); Pair result = convertBytes(bytes, Math.min(bom == null ? 0 : bom.length, bytes.length), bytes.length, internalCharset); @@ -600,9 +586,7 @@ public final class LoadTextUtil { boolean saveBOM, @NotNull FileType fileType, @NotNull NullableConsumer fileTextProcessor) { - Charset initialCharset = EncodingManager.getInstance().getEncoding(virtualFile, true); - Trinity - info = doDetectCharsetAndSetBOM(virtualFile, bytes, startOffset, endOffset, saveBOM, fileType, initialCharset, __->null); + Trinity info = detectInternalCharsetAndSetBOM(virtualFile, bytes, startOffset, endOffset, saveBOM, fileType); Charset internalCharset = info.getFirst(); CharsetToolkit.GuessedEncoding guessed = info.getSecond(); if (internalCharset == null || guessed == CharsetToolkit.GuessedEncoding.BINARY || guessed == CharsetToolkit.GuessedEncoding.INVALID_UTF8) { diff --git a/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java b/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java index ec96763c2f37..19a03b879b30 100644 --- a/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java +++ b/platform/platform-impl/src/com/intellij/openapi/fileTypes/impl/FileTypeManagerImpl.java @@ -60,6 +60,7 @@ import com.intellij.util.concurrency.AppExecutorUtil; import com.intellij.util.concurrency.BoundedTaskExecutor; import com.intellij.util.containers.ConcurrentPackedBitsArray; import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.containers.HashSetQueue; import com.intellij.util.io.URLUtil; import com.intellij.util.messages.MessageBus; import com.intellij.util.messages.MessageBusConnection; @@ -76,8 +77,6 @@ import java.io.*; import java.net.URL; import java.nio.channels.FileChannel; import java.util.*; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -253,8 +252,10 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent log("F: after() queued to redetect: " + files); } - if (filesToRedetect.addAll(files)) { - awakeReDetectExecutor(); + synchronized (filesToRedetect) { + if (filesToRedetect.addAll(files)) { + awakeReDetectExecutor(); + } } } } @@ -351,20 +352,23 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent } private final BoundedTaskExecutor reDetectExecutor = new BoundedTaskExecutor("FileTypeManager redetect pool", PooledThreadExecutor.INSTANCE, 1, this); - private final BlockingQueue filesToRedetect = new LinkedBlockingDeque<>(); + private final HashSetQueue filesToRedetect = new HashSetQueue<>(); + private static final int CHUNK_SIZE = 10; private void awakeReDetectExecutor() { - reDetectExecutor.submit(new Runnable() { - private static final int CHUNK = 10; - @Override - public void run() { - List files = new ArrayList<>(); - int drained = filesToRedetect.drainTo(files, CHUNK); - reDetect(files); - if (drained == CHUNK) { - awakeReDetectExecutor(); + reDetectExecutor.submit(() -> { + List files = new ArrayList<>(CHUNK_SIZE); + synchronized (filesToRedetect) { + for (int i = 0; i < CHUNK_SIZE; i++) { + VirtualFile file = filesToRedetect.poll(); + if (file == null) break; + files.add(file); } } + if (files.size() == CHUNK_SIZE) { + awakeReDetectExecutor(); + } + reDetect(files); }); } @@ -381,7 +385,9 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent @TestOnly @NotNull Collection dumpReDetectQueue() { - return new ArrayList<>(filesToRedetect); + synchronized (filesToRedetect) { + return new ArrayList<>(filesToRedetect); + } } @TestOnly @@ -501,6 +507,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent return getFileTypeByFileName((CharSequence)fileName); } + @Override @NotNull public FileType getFileTypeByFileName(@NotNull CharSequence fileName) { FileType type = myPatternsTable.findAssociatedFileType(fileName); @@ -765,80 +772,77 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent return file.getFileSystem() instanceof FileSystemInterface; } - private boolean processFirstBytes(@NotNull final InputStream stream, final int length, @NotNull Processor processor) throws IOException { + /** + * Read first {@code firstChunkLength} bytes from the {@code stream} to pass them to the {@code processor}. + * {@code processor} may later request all the bytes from the stream by calling {@code Getter.get()} + */ + private void processFirstBytes(@NotNull InputStream stream, int fileLength, int firstChunkLength, @NotNull PairConsumer> processor) throws IOException { final byte[] bytes = FileUtilRt.getThreadLocalBuffer(); - assert bytes.length >= length : "Cannot process more than " + bytes.length + " in one call, requested:" + length; + assert bytes.length >= firstChunkLength : "Cannot process more than " + bytes.length + " in one call, requested:" + firstChunkLength; - int n = stream.read(bytes, 0, length); + int n = readSafely(stream, bytes, 0, firstChunkLength); + if (n<=0) return; + + ByteSequence firstChunk = new ByteSequence(bytes, 0, n); + + processor.consume(firstChunk, ()->{ + if (fileLength <= n) { + // the file is small, fit into the first chunk + return firstChunk; + } + byte[] buffer = bytes.length >= fileLength ? bytes : ArrayUtil.realloc(bytes, fileLength); + int read; + try { + read = readSafely(stream, buffer, n, fileLength - n); + } + catch (IOException e) { + return null; + } + if (read <= 0) return null; + return new ByteSequence(buffer, 0, n+read); + }); + } + + private int readSafely(InputStream stream, byte[] buffer, int offset, int length) throws IOException { + int n = stream.read(buffer, offset, length); if (n <= 0) { // maybe locked because someone else is writing to it // repeat inside read action to guarantee all writes are finished if (toLog()) { log("F: processFirstBytes(): inputStream.read() returned "+n+"; retrying with read action. stream="+ streamInfo(stream)); } - n = ApplicationManager.getApplication().runReadAction((ThrowableComputable)() -> stream.read(bytes, 0, length)); + n = ApplicationManager.getApplication().runReadAction((ThrowableComputable)() -> stream.read(buffer, offset, length)); if (toLog()) { log("F: processFirstBytes(): under read action inputStream.read() returned "+n+"; stream="+ streamInfo(stream)); } - if (n <= 0) { - return false; - } } - - return processor.process(new ByteSequence(bytes, 0, n)); + return n; } @NotNull private FileType detectFromContentAndCache(@NotNull final VirtualFile file) throws IOException { long start = System.currentTimeMillis(); Ref result = new Ref<>(UnknownFileType.INSTANCE); - boolean r = false; InputStream inputStream = ((FileSystemInterface)file.getFileSystem()).getInputStream(file); try { if (toLog()) { log("F: detectFromContentAndCache(" + file.getName() + "):" + " inputStream=" + streamInfo(inputStream)); } - r = processFirstBytes(inputStream, DETECT_BUFFER_SIZE, byteSequence -> { - // use PlainTextFileType because it doesn't supply its own charset detector - // help set charset in the process to avoid double charset detection from content - LoadTextUtil.processTextFromBinaryPresentationOrNull(byteSequence.getBytes(), - byteSequence.getOffset(), byteSequence.getOffset()+byteSequence.getLength(), - file, true, true, - PlainTextFileType.INSTANCE, (@Nullable CharSequence text) -> { - FileTypeDetector[] detectors = Extensions.getExtensions(FileTypeDetector.EP_NAME); - if (toLog()) { - log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): byteSequence.length=" + byteSequence.getLength() + - "; isText=" + (text != null) + "; text='" + (text == null ? null : StringUtil.first(text, 100, true)) + "'" + - ", detectors=" + Arrays.toString(detectors)); - } - FileType detected = null; - for (FileTypeDetector detector : detectors) { - try { - detected = detector.detect(file, byteSequence, text); - } - catch (Exception e) { - LOG.error("Detector " + detector + " (" + detector.getClass() + ") exception occurred:", e); - } - if (detected != null) { - if (toLog()) { - log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): detector " + detector + " type as " + detected.getName()); - } - break; - } - } - - if (detected == null) { - detected = text == null ? UnknownFileType.INSTANCE : PlainTextFileType.INSTANCE; - if (toLog()) { - log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): " + - "no detector was able to detect. assigned " + detected.getName()); - } - } - result.set(detected); - }); - - return true; + processFirstBytes(inputStream, (int)file.getLength(), DETECT_BUFFER_SIZE, (firstChunk,entireFileBytesGetter) -> { + detect(file, result, firstChunk); + if (result.get() == UnknownFileType.INSTANCE) { + // detected as binary + return; + } + int firstChunkLength = firstChunk.getLength(); + // It seems the file is text but the problem is we might have detected its charset wrong + // The first DETECT_BUFFER_SIZE bytes might have been not enough to e.g. encounter some specific UTF-8 byte sequences + // Need to scan all the file text + ByteSequence entireFile = entireFileBytesGetter.get(); + if (entireFile != null && entireFile.getLength() != firstChunkLength) { + detect(file, result, entireFile); + } }); } finally { @@ -848,7 +852,6 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent byte[] buffer = new byte[50]; int n = newStream.read(buffer, 0, buffer.length); log("F: detectFromContentAndCache(" + file.getName() + "): result: " + result.get().getName() + - "; processor ret: " + r + "; stream: " + streamInfo(inputStream) + "; newStream: " + streamInfo(newStream) + "; read: " + n + @@ -874,6 +877,46 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent return fileType; } + private void detect(@NotNull VirtualFile file, Ref result, ByteSequence byteSequence) { + // use PlainTextFileType because it doesn't supply its own charset detector + // help set charset in the process to avoid double charset detection from content + LoadTextUtil.processTextFromBinaryPresentationOrNull(byteSequence.getBytes(), + byteSequence.getOffset(), byteSequence.getOffset()+byteSequence.getLength(), + file, true, true, + PlainTextFileType.INSTANCE, (@Nullable CharSequence text) -> { + FileTypeDetector[] detectors = Extensions.getExtensions(FileTypeDetector.EP_NAME); + if (toLog()) { + log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): byteSequence.length=" + byteSequence.getLength() + + "; isText=" + (text != null) + "; text='" + (text == null ? null : StringUtil.first(text, 100, true)) + "'" + + ", detectors=" + Arrays.toString(detectors)); + } + FileType detected = null; + for (FileTypeDetector detector : detectors) { + try { + detected = detector.detect(file, byteSequence, text); + } + catch (Exception e) { + LOG.error("Detector " + detector + " (" + detector.getClass() + ") exception occurred:", e); + } + if (detected != null) { + if (toLog()) { + log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): detector " + detector + " type as " + detected.getName()); + } + break; + } + } + + if (detected == null) { + detected = text == null ? UnknownFileType.INSTANCE : PlainTextFileType.INSTANCE; + if (toLog()) { + log("F: detectFromContentAndCache.processFirstBytes(" + file.getName() + "): " + + "no detector was able to detect. assigned " + detected.getName()); + } + } + result.set(detected); + }); + } + // for diagnostics @SuppressWarnings("ConstantConditions") private static Object streamInfo(InputStream stream) throws IOException { @@ -1297,7 +1340,7 @@ public class FileTypeManagerImpl extends FileTypeManagerEx implements Persistent } StringTokenizer tokenizer = new StringTokenizer(semicolonDelimited, FileTypeConsumer.EXTENSION_DELIMITER, false); - ArrayList list = new ArrayList<>(); + ArrayList list = new ArrayList<>(semicolonDelimited.length() / "py;".length()); while (tokenizer.hasMoreTokens()) { list.add(new ExtensionFileNameMatcher(tokenizer.nextToken().trim())); } diff --git a/platform/platform-tests/testData/vfs/encoding/BIGCHANGES b/platform/platform-tests/testData/vfs/encoding/BIGCHANGES new file mode 100644 index 000000000000..ecc2e433ca6b --- /dev/null +++ b/platform/platform-tests/testData/vfs/encoding/BIGCHANGES @@ -0,0 +1,3186 @@ +Platform 1.59 + +* We improved the error responses generated when parsing JSON fails. + +Platform 1.58 + +* BalancingHttpClient + + The BalancingHttpClient now implements retry budgets. Over a period + configured from the "http-client.retry-budget.ratio-period" configuration + option (default 10s) the number of retries attempted will be limited to a + ratio (configured from "http-client.retry-budget.ratio") of requests + submitted. A number of retries per second (configured from + "http-client.retry-budget.min") will additionally be permitted. + + The number of retries that are not attempted due to the budget will be + reported using the HttpClient.QUALIFIER.RetryBudgetExhausted.Count + metric. + + This feature is currently effectively disabled with large configured ratios + and minimums. Recommended values are 0.2 for ratio and 10 for minimum. + This feature should only be enabled in combination with failure accrual. + +* [Bug] The reporting client catches and logs exceptions thrown while + collecting metrics. + +* [Bug] Cleaned up configuration warnings exposed by the refactor in the + previous release. + +* Library Upgrades + + - logback to 1.2.3 (was 1.1.8) + +Platform 1.57 + +* BalancingHttpClient + + The HttpServiceBalancer now implements failure accrual. Destinations + that fail than a configurable number of times in a row will be removed from + consideration for a configurable backoff. This is currently disabled by + default and is configured with the + "service-balancer.SERVICE.consecutive-failures" configuration option. A + value of "5" is recommended. + +* Discovery + + We created a StaticDiscoveryModule which does not talk to the discovery + service. It instead obtains service location from configuration, using + a comma-separated list of URIs from the "service-balancer.SERVICETYPE.uri" + config option. + +* Configuration + + Warnings from config objects built by a ConfigurationAwareProvider are now + reported. Such objects are also enumerated by ConfigurationInspector and + thus the corresponding configuration is logged upon startup. + + In order to permit a ConfigurationAwareProvider to inspect the application + modules, we added a buildConfigObjects(Iterable) method. + + We removed ConfigurationAwareProvider.setWarningsMonitor(). + + We deprecated the + ConfigurationValidator(ConfigurationFactory, WarningsMonitor) constructor. + Use the new ConfigurationValidator(ConfigurationFactory) constructor. + + We deprecated the ConfigRecord.createConfigRecord(ConfigurationFactory) + static factory method. Callers should not be creating their own ConfigRecord + objects. + + ConfigRecord.getKey() can now return null and + ConfigRecord.getComponentName() can now return the empty string + (both if the config object was built by a ConfigurationAwareProvider). + +* Library Upgrades + + - Guava to 22.0 (from 21.0) + - [Bug] Fix LocalCache.compute() deadlock issue + - HostAndPort.getHostText() removed (use HostAndPort.getHost()) + +Platform 1.56 + +* HttpClient and HttpServer + + [Bug] In some cases the Jetty HttpClient.execute() method would leave the + request in progress after returning. + + HttpServer no longer sets the blocking timeout. That was an ineffective + workaround. + +* BalancingHttpClient + + The BalancingHttpClient now implements backoff between retries using + decorrelated jitter. The minimum and maximum delays between attempts + are configurable. + + We created HttpClientBinder.bindBalancingHttpClient(String, Annotation, + String, Set) and bindBalancingHttpClient(String, Annotation, String, + Key) methods for binding static or custom + BalancingHttpClients to a parameterized Annotation. + +* Logging + + We added Level.TRACE and Level.ALL log levels. + +* JAX-RS + + We added JaxrsModule.adminOnlyJaxrsModule() for servers that want to provide + their own custom servlet but still have the admin port JAX-RS services. + +* Library Upgrades + + - Jetty to 9.3.19 (from 9.3.15) + +Platform 1.55 + +* RPM packaging + + The rpm-maven-plugin upgrade in 1.53 created RPMs with an empty classifier. + We restored the previous classifier of "rpm". + +* Library Upgrades + + - Jackson to 2.7.9 (was 2.6.4) + - jackson-databind to 2.7.9.1 (fixes deserialization vulnerability + when using @JsonTypeInfo or "default typing") + - ObjectMapper default timezone now UTC, not GMT + - JsonInclude.Include.NON_EMPTY now only applies to Collection, Map, + array, and String (not default scalar values) + +Platform 1.54 + +* Configuration + + We improved the error message when a required configuration property is + missing. + +* Trace tokens + + Since the trace token is (as of Platform 1.43) included in the thread name, + it was being included in the launcher.log twice. We removed it from its old + location. + +* [Bug] Set the blocking timeout to be more than the idle timeout. + +* [Bug] Fix the blocking timeout to apply independently to each read call. + +Platform 1.53 + +* Library Upgrades + + - commons-math removed (was 2.2) + +* Maven plugin upgrades + + - rpm-maven-plugin 2.1.5 (from 2.1-alpha-4) + Configuration changed to make RPM version and release attributes available + as properties. + +Platform 1.52 + +* DiscoveryBinder.bindHttpAnnouncement() will include the admin port URL in a + new "admin" property of the service announcement. + +* JAX-RS + + JAX-RS resource method parameters may now be annotated with + @com.proofpoint.reporting.Key annotations. The metrics generated + by such method calls will then be broken down by the toString of the + corresponding parameter values, similar to a report collection. + +* Reporting + + We created TestingReportingModule for unit tests that need to test the + reported metrics that are generated. It provides a ReportingTester that + can be used to collect and return reported metrics. + +* HttpClient + + The HttpClient.IoPool.*.FreeThreadCount metric now uses the HttpClient's + config prefix, with lower-hyphen converted to upper-camel, instead of the + qualifer as the third component in its name. This was because previously + multiple BalancingHttpClients would generate multiple metrics with the same + name, leading to exceptions. + +* [Bug] Fixed the Platform-provided ExceptionMapper classes to not pick up + media types from the resource method annotations. + +Platform 1.51 + +* Trace tokens + + We extended TraceTokenManager to support adding additional properties to the + trace token state. To this end, we created a new TraceToken class and + deprecated TraceTokenManager.getCurrentRequestToken() in favor of the new + TraceTokenManager.getCurrentTraceToken(). + + We changed the format of trace token ids created by the HTTP server so that + they start with an encoding of the server's and client's IPs. + +* We increased the default HttpClient connection timeout to 2 seconds. + +* We suppress logging the thread dump when Bootstrap is in quiet mode. + +* [Bug] The HttpServer.BusyThreads.Max metric leaked counts for calls + that threw uncaught exceptions. + +* Library Upgrades + + - Guava to 21.0 (from 20.0) + - Objects.firstNonNull() and Objects.toStringHelper() removed + (use MoreObjects) + - MoreExecutors.sameThreadExecutor() removed + (use directExecutor() or newDirectExecutorService()) + - MapConstraint and MapConstraints removed + +Platform 1.50 + +* Deprecation removals + + We removed the following deprecated features: + + - @ConfigMap + - ConfigurationModule.registerConfigurationClasses(Module) + - The TestingHttpClient constructors taking Function + - The public TestingResponse constructors + - TestingResponse.mockResponse(HttpStatus, MediaType, String) + - AsyncHttpClientModule + - HttpClientAsyncBindingBuilder + - The unused http-client.keep-alive-interval config option + +* Version numbers + + We added the application and Platform version numbers to NodeInfo. + + The ReportCollector.ServerStart and ReportCollector.NumMetrics metrics + now carry applicationVersion and platformVersion tags. + + The admin port now has an /admin/version resource which returns the + application name, version, and Platform version. + +* We added an /admin/jstack resource on the admin port to dump the thread + stacks. + +* Bootstrap + + Bootstrap now sets a default uncaught exception handler that logs the + exception. + +* Events + + The event client now supports sending events that are implemented with + AutoValue (or other subclasses of the event class). + +Platform 1.49 + +* HttpServer sets the blocking timeout in order to work around a Jetty bug. + +* Library Upgrades + + - Jetty to 9.3.15 (from 9.3.14) + - [Bug] fixes deadlocks in HttpServer + +Platform 1.48 + +* We dump the thread stacks upon shutdown. + +* Bootstrap + + We created Bootstrap.bootstrapTest() to simplify using Bootstrap in unit + tests. We deprecated Bootstrap.setRequiredConfigurationProperty() and + Bootstrap.setRequiredConfigurationProperties() when not using + Bootstrap.bootstrapTest(). + +* HttpClient + + HttpClient implementations now forward trace tokens by default. We added + a .withoutTracing() method to the binding builders to disable this and + deprecated the .withTracing() method. + + We added an HttpClient.IoPool.Shared.FreeThreadCount metric to track + the count how much unused thread capacity is in the shared client IO pool. + There are also corresponding metrics for private client IO pools. + + We changed the BalancingHttpClient to call the ResponseHandler with the + Request as made to the selected instance instead of the Request that was + passed to the BalancingHttpClient. That way, the ResponseHandler may inspect + the Request URI to determine which instance was used. In the case where the + HttpServiceBalancer threw an exception (such as ServiceUnavailableException) + the BalancingHttpClient will call the ResponseHandler with the Request that + was passed to the BalancingHttpClient. + +* JAX-RS + + We disabled processing of query parameters as form parameters for resources + that consume application/x-www-form-urlencoded content. We added a + configuration option "jaxrs.query-params-as-form-params" for re-enabling + this behavior. + +* We fixed warnings caused by how the library pom was invoking the Surefire + plugin. + +* [Bug] Ensure LifeCycleManager shutdown methods can log. + +* [Bug] Catch and log exceptions thrown by LifeCycleManager shutdown methods. + +* Library Upgrades + + - bval-jsr 1.1.2 (from bval-jsr303 0.5) + - Code will need to change any references to the org.apache.bval.jsr303 + package to org.apache.bval.jsr + - mockito to 2.2.28 (from 1.10.8) + - Hamcrest matchers, such as Matchers.argThat(), have moved to + MockitoHamcrest (MockitoHamcrest.argThat()) + - anyX() and any(SomeType.class) matchers now reject nulls and check type. + - testng to 6.10 (from 6.9.6) + - auto-value to 1.3 (from 1.1) + - logback to 1.1.8 (from 1.1.7) + +Platform 1.47 + +* We made the server refuse to start if the discovery client is configured to + announce an HTTPS service with an unqualified hostname. + +* We added TraceTokenCopyingExecutor, which wraps an ExecutorService to copy + trace tokens into the destination thread for the duration of the call. + +* HttpClient will periodically log a dump of its state upon getting a + ClosedByInterruptException. + +* Library Upgrades + + - Jetty to 9.3.14 (from 9.3.12) + - Guava to 20.0 (from 19.0) + +Platform 1.46 + +* We disabled JAX-RS resolution of relative URIs in the Location header, as + it was preventing redirecting to URIs with less-than-strict quoting + +Platform 1.45 + +* Reverted bval-jsr 1.1.1 back to bval-jsr303 0.5 + +Platform 1.44 + +* JAX-RS + + The JaxrsBinder EDSL now supports a .withApplicationPrefix() method which + causes the reported metrics to have names prefixed with the application + name. + + We deprecated the nonsensical JaxrsBinder.bind(TypeLiteral), + JaxrsBinder.bind(Key), and their corresponding bindAdmin methods. + +* HTTP/2 support + + The HTTP server now has experimental support for the HTTP/2 + protocol for HTTP only (not HTTPS). The server supports the HTTP/2 + cleartext upgrade mechanism (h2c) which allows running both versions + of the protocol on the same port. Both versions are always enabled. + +* HTTP Client API changes for Case-Insensitive Headers + + HTTP header field names are always lowercase in HTTP/2. They should + always be treated as case insensitive, but the previous HTTP client + API made that difficult by returning a map of strings. + + The .getHeaders() method of the Response interface and various + ResponseHandler response objects, as well as the object constructors + now use the new case-insensitive HeaderName class. Additionally, + these objects now have a getHeaders() method that takes a field name + and returns a list of field values. + +* Logging + + We have created new configuration options "log.max-total-size" and + "http-server.log.max-total-size" for limiting the total amount of + space taken by archived log files. + + As part of this change, we deprecated the existing Logging.logToFile() and + Logging.createFileAppender() methods for versions which take this new option + as an additional parameter. + +* HttpClient + + HttpClient will periodically log a dump of its state upon getting a + RejectedExecutionException. + + We added configuration options for specifying the certificate trust store. + +* Configuration + + We added a AbstractConfigurationAwareModule.build(Class, String) method + for building prefixed config objects. + +* Library Upgrades + + - Jetty to 9.3.12 (from 9.3.11) + - bval-jsr 1.1.1 (from bval-jsr303 0.5) + - logback to 1.1.7 (from 1.1.5) + +Platform 1.43 + +* We now disable use of the /admin/stop-announcing resource on the admin port + when HTTPS is not enabled. + +* We modified TraceTokenManager to put the trace token in the thread name. + +* We disabled the following TLS ciphers: + + - TLS_RSA_WITH_AES_256_CBC_SHA256 + - TLS_RSA_WITH_AES_128_CBC_SHA256 + - TLS_RSA_WITH_AES_256_GCM_SHA384 + - TLS_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA + - SSL_RSA_WITH_3DES_EDE_CBC_SHA + +* JAX-RS + + We added support for a ClientInfo @Context parameter for getting the client + IP address, following the X-Forwarded-For: header. + + We created the @AccessDoesNotRequireAuthentication annotation for use by + JAX-RS request filters which would otherwise require authentication to + access a JAX-RS resource. We annotated several Platform-provided JAX-RS + resources with this annotation. + +* We deprecated HttpRequestEvent. + +* Library Upgrades + + - Jetty to 9.3.11 (from 9.3.7) + +Platform 1.42 + +* Due to reports of random client request corruption, we reverted jetty + to 9.3.7. + +* We reverted the .withApplicationPrefix() method of the JaxrsBinder EDSL. + While this change was source compatible, it was not binary compatible. + This change will be made in a later version of Platform. + +Platform 1.41 + +* We re-enabled http-server support for the SSLv2Hello protocol that Jetty + disabled in Platform 1.39. + +* AsyncSemaphore now uses JDK Function instead of Guava Function. This change + is generally source-compatible but not binary-compatible. + +* HttpServer + + We replaced the deprecated GzipFilter with GzipHandler. + +* HttpClient + + We added a FullJsonResponseHandler.getResponseSize() method. + +* We removed all usage of ThreadGroup. + +* Library Upgrades + + - jmxutils to 1.19 (from 1.18) fixes the server not starting when LogJmxModule + is used due to a Java 8 compatibility issue with jmxutils. + - Jersey to 2.23.1 (from 2.22) + +Platform 1.40 + +* We re-enabled the SHA1 ciphers that Jetty disabled in the http client + in Platform 1.39. + +* Logging configuration + + We added the "http-server.log.enabled" configuration option for disabling + the HttpServer request log. + + We renamed the "log.output-file" configuration option to "log.path" to make + it consistent with "http-server.log.path". + +Platform 1.39 + +* Library Upgrades + + - Jetty to 9.3.11 (from 9.3.10) + +Platform 1.38 + +* JAX-RS + + We deprecated the implicit JaxrsModule() constructor. + Use JaxrsModule.explicitJaxrsModule() and explicitly bind JAX-RS resources + using JaxrsBinder. + + The JaxrsBinder EDSL now supports a .withApplicationPrefix() method which + causes the reported metrics to have names prefixed with the application + name. + +* HttpClient + + We improved the buffering of data from DynamicBodySource, reducing the + overhead of chunked encoding. + + We added a BodySource.getLength() method, permitting DynamicBodySource + and InputStreamBodySource implementations to specify the content length + up front. + +* [Bug] We allow '-' characters in the node.hostname config property. + +* Library Upgrades + + - Jetty to 9.3.10 (from 9.3.7) + +Platform 1.37 + +* We changed the logging of configuration upon startup to sort by property. + +* We modified the launcher to refuse to start as root. + +Platform 1.36 + +* We made AdminServerCredentialVerifier public. + +* Maven release plugin + + We have reconfigured the maven-release-plugin to run tests during the + release:prepare phase. Projects that configure the maven-release-plugin + might need to adjust that configuration to account for these changes to + the library POM. + +* Fixed use of deprecated ignoredResources duplicate-finder-maven-plugin + configuration option. + +* [Bug] LoggingOutputStream incorrectly treated log lines as format strings. + +Platform 1.35 + +* Maven plugin upgrades + + We have upgraded from the com.ning.maven.plugins + maven-duplicate-finder-plugin 1.0.6 to the org.basepom.maven + duplicate-finder-maven-plugin 1.2.1. Any client pom files that adjust the + duplicate-finder configuration will need to do so using the new groupId and + artifactId. + + - maven-deploy-plugin 2.8.2 (from 2.7) + - maven-clean-plugin 3.0.0 (from 2.5) + - maven-install-plugin 2.5.2 (from 2.4) + - build-helper-maven-plugin 1.10 (from 1.8) + - maven-enforcer-plugin 1.4.1 (from 1.2) + - maven-compiler-plugin 3.5.1 (from 3.0) + - maven-assembly-plugin 2.6 (from 2.5.3) + - maven-surefire-plugin 2.19.1 (from 2.14) + - maven-release-plugin 2.5.3 (from 2.5) + - maven-jar-plugin 2.6 (from 2.4) + - maven-source-plugin 3.0.0 (from 2.2.1) + - findbugs-maven-plugin 3.0.3 (from 3.0.1) + - jacoco-maven-plugin 0.7.6 (from 0.7.4) + - maven-site-plugin 3.5.1 (from 3.2) + - maven-gpg-plugin 1.6 (from 1.4) + - maven-shade-plugin 2.4.3 (from 2.3) + - maven-antrun-plugin 1.8 (from 1.7) + +* [Bug] Allow lifecycle.stop-traffic.delay config option to be specified + in config.properties. + +* [Bug] The request.log omitted the URI query string. + +Platform 1.34 + +* We demoted a dependency from the log module to the testing module to test + scope. Projects that were inadvertently using this dependency to pull in + the testing module will need to add their own dependency on testing in their + pom.xml + +* Health checks + + We added a @HealthCheckRemoveFromRotation annotation, which may be placed + on either a method with no arguments or a field of type AtomicReference. + When the object is bound with HealthBinder, the method will be called or + the field will be examined by the "/inrotation.txt" resource. A value of + null indicates healthy; any other value indicates a critical problem, with + the toString() used as the message. + +* Reporting + + Objects exported to the reporting subsystem (using ReportBinder, + ReportExporter, or report collections) are no longer also exported to JMX. + If you need them to continue to be exported to JMX, add corresponding calls + to either ExportBinder or MBeanExporter. + + We have added easier to use ReportBinder.export() and + ReportBinder.bindReportCollection() EDSLs. See the + ReportBinder class JavaDoc for details. We have deprecated the + ObjectName-based .as() methods. + + We have deprecated the .withGeneratedName() methods of the + ReportBinder.export() and ReportBinder.bindReportCollection() EDSLs as + they are no longer necessary. + + We have added an easier to use ReportExporter.export() method (and + corresponding ReportExporter.unexportObject() method). See the method + JavaDoc for details. We have deprecated the ObjectName-based + ReportExporter.export() and ReportExporter.unexport() methods. + + We have added support for prefixing the metric name with the application + name. + + We have added an easier to use + ReportCollectionFactory.createReportCollection() method. See the method + JavaDoc for details. We have deprecated the ObjectName-based + ReportCollectionFactory.createReportCollection() method. + + We added support for report collection methods which take no arguments. + The singleton object returned from such methods is constructed and exported + when the report collection implementation is created. Such methods should + normally be declared to return the non-sparse stats objects. + + We have added to TestingReportCollectionFactory .getArgumentVerifier(T) + and .getReportCollection(T) methods that take the testing report collection + returned from .createReportCollection(...) as their argument. These are now + the preferred methods--the previous .getArgumentVerifier(Class) and + .getReportCollection(Class) methods have been deprecated. + +* Packaging + + Client projects that want to create a distribution tar file may do so by + creating a ".build-distribution" file and inheriting directly or indirectly + from the library pom, instead of inheriting directly from the + rest-server-base pom. Client projects that want to create an rpm may now + do so by inheriting directly or indirectly from the library pom (and + creating a ".build-rpm" file). + +* We re-enabled the RSA-SHA1 ciphers that Jetty disabled in the http client + in Platform 1.32. + +* [Bug] We fixed the skeleton server archetype to provide a blank README.txt + +Platform 1.33 + +* RPM Packaging + + In order to support CentOS 7, the RPM created for a Platform service now + includes a systemd service file. + +Platform 1.32 + +* Bootstrap + + We added support to LifeCycleManager for a new @StopTraffic annotation. Upon + server shutdown, methods annotated with @StopTraffic are called first, then + shutdown waits for the duration in the "lifecycle.stop-traffic.delay" + configuration option (default "0s"), then shutdown continues with calling + methods annotated with @PreDestroy. + + We changed the Discovery service de-announcement to @StopTraffic. + +* Reporting + + reportBinder.bindReportCollection() can now bind a report collection to an + annotation. + + Report collections now handle java.util.Optional values. + +* Maven + + We have configured maven-duplicate-finder-plugin to ignore duplicate + javac.sh resources from Jetty packages. + +* Library Upgrades + + - Jetty to 9.3.7 (from 9.3.6) + - SLF4J to 1.7.16 (from 1.7.12) + - Logback to 1.1.5 (from 1.1.3) + +Platform 1.31 + +* The following deprecated features have been removed: + + - TraceTokenModule + - The TraceTokenManager constructor has been made private. + - BodyGenerator + - StaticBodyGenerator implements BodySource instead of BodyGenerator. + - Request.getBodyGenerator() + - EventClient.post(EventGenerator), EventGenerator, and EventPoster. + - NodeInfo.getBinarySpec(), NodeInfo.getConfigSpec(), + NodeInfo.getInstanceId(), and the NodeInfo constructor that takes + a binarySpec and configSpec. + - The "http-client.max-connections" and "http-server.log-retention-time" + configuration options. + - The "etc/log.config" file. Use "etc/log.properties" instead. + + Additionally, Request.Builder.setBodyGenerator(BodyGenerator) changed to + Request.Builder.setBodyGenerator(StaticBodyGenerator) (and remains + deprecated). + +Platform 1.30 + +* Skeleton server + + We have updated the skeleton server to run HTTPS by default. + +* Library Upgrades + + - Guava to 19.0 + - Jackson to 2.6.4 + +Platform 1.29 + +* HttpClient + + The "http-client.max-requests-queued-per-destination" configuration option + may now be set to the value "0". This permits as many outstanding requests + as configured by the "http-client.max-connections-per-server" option, but + will reject new requests once all those connections have been assigned + requests. + +* Library Upgrades + + - Jetty to 9.3.6 + +Platform 1.28 + +* JAX-RS + + We disabled on the main service port Jersey's automatic generation of WADL + and default support of the OPTIONS method. + + We added an /admin/wadl resource to the admin port which returns the server + port's WADL. + +* Configuration + + Configuration objects are no longer treated as singletons, in order + to avoid action at a distance. + + We deprecated ConfigurationFactory.registerConfigurationClasses(Module) + +* HttpServer + + We created the http-server.stop-timeout configuration option to allow + changing the amount of time upon server shutdown that requests are allowed + to complete before being interrupted. + + We created new defaults for existing configuration options: + + http-server.https.keystore.path=etc/keystore.jks + http-server.https.keystore.key=keystore + +* Launcher + + The amount of time the launcher will wait for a server to gracefully stop + before sending it a SIGKILL is now configurable through the + launcher.stop-timeout-seconds system property. The default is "60". + +* We added a "ci" Maven profile that builds Javadoc. + +* We removed the dbpool sub-project. + +* Library Upgrades + + - Jetty to 9.2.13 + - Jersey to 2.22 + - slf4j to 1.7.12 + - joda-time to 2.9 (Fixes a compatibility issue with Java 8u60.) + - testng to 6.9.6 + - Jacoco to 0.7.4 + +Platform 1.27 + +* TestingResponse + + We created a new builder for TestingResponse and deprecated most of + the previous methods of creating TestingResponse. For example: + + TestingResponse.mockResponse() + .header("X-Foo", "bar") + .jsonBody(object) + .build(); + +* Skeleton server and Sample server + + The Maven archetypes now supply a gitignore file. This file should be + renamed to ".gitignore" (it is impractical to get an archetype to create + a file named ".gitignore"). + + We have updated the skeleton and sample servers to reflect current practice. + + - The skeleton server Main class includes code to override the default value + of http-server.http.port. + - TestServer initializes and tears down the HttpClient once for the class. + - Resource classes are tested only through TestServer. + - TestServer no longer uses resource files to supply JSON. + - The sample server no longer uses events + +* Admin port + + We added an /admin/configuration resource to the admin port which permits + retrieving the redacted configuration settings. + +* HttpServer + + We created TestingAdminHttpServer and TestingAdminHttpServerModule for + testing of admin port resources. + +* Launcher + + We added the "-XX:HeapDumpPath=var" option to the set of default JVM options. + +* [Bug] Fix connection hang when ResponseHandler throws without reading large + response. + +* Library Upgrades + + - AutoValue to 1.1 + +* Maven plugin upgrades + - maven-resources-plugin to 2.7 + - maven-invoker-plugin to 2.0.0 + +Platform 1.26 + +* HttpServer + + We added a HttpServer.BusyThreads.Max metric that reports the maximum number + of busy threads in the http-server pool. + +* Library Upgrades + + - Jersey to 2.21 + +Platform 1.25 + +* BalancingHttpClient + + The balancer now reports a ServiceClient.Concurrency.Max metric that reports + the maximum number of concurrent requests to any instance in the pool. + +* Stats + + We created a MaxGauge stats class for reporting the maximum value, over the + one minute reporting interval, value of a metric. + +Platform 1.24 + +* Logging + + Platform applications with log.output-file configured now also write a + bootstrap.log file in the same directory. The bootstrap.log file contains + a copy of messages logged from Bootstrap and is truncated on every startup. + This is intended to make the (redacted) configuration and other startup + information more accessible when troubleshooting. + + We added Logging.addLogTester() and Logging.resetLogTesters() methods for + writing unit tests that verify logging. + +* TestingHttpClient + + We created a default constructor for TestingHttpClient. The constructed + TestingHttpClient needs a call to setProcessor(Processor) before it can + be effectively used. + + We added TestingHttpClient.getRequestCount(). + +* TestingTicker and SerialScheduledExecutorService + + We added TestingTicker.elapseTime() and deprecated TestingTicker.increment() + for consistency with SerialScheduledExecutorService. + + We added elapseTimeNanosecondBefore() methods to TestingTicker and + SerialScheduledExecutorService. + +* Library Upgrades + + - jnr-posix to 3.0.17 + +Platform 1.23 + +* BalancingHttpClient + + When a ResponseHandler throws an exception, the ServiceClient.Failure.Count + metric incremented is given an additional "handlerException" tag with a + value of that exception's class. + +* [Bug] Fixed the request log's logic for when to use addresses in the + X-Forwarded-For: header. + + - Ignore X-Forwarded-For: if the connection is from an external IP address. + - Use the closest (last-listed) external IP address, if any. + - Use the furthest (first-listed) IP address if all addresses are internal. + +* Library Upgrades + + - Jetty to 9.2.12 + +Platform 1.22 + +* BalancingHttpClient + + We a created HttpClientBinder.bindBalancingHttpClient(String name, + Class annotation, Set baseUris) method for + binding a BalancingHttpClient with a fixed set of destination URIs. + + We a created HttpClientBinder.bindBalancingHttpClient(String name, + Class annotation, + Key balancerKey) method for + binding a BalancingHttpClient with a custom HttpServiceBalancer. + + We added 499, 598, and 599 to the set of retryable response codes. + +* Library Upgrades + + - Joda-time to 2.8 + +Platform 1.21 + +* DHE disabled + + We disabled all default-enabled DHE ciphers in both HttpServer and + JettyHttpClient. + +* Future Utilities + + We've added MoreFutures class which contains utility methods for Future and + the new Java8 CompletableFuture. For basic future, there are methods for + fetching the future value with minimal exception handling. For + CompletableFuture, there are methods for easily creating futures. Finally, + we have added methods for converting to and from Guava ListenableFuture to + ease the transition to CompletableFuture. + +* HttpServer + + We changed the default value of the "http-server.accept-queue-size" + configuration option to 8000. + +* Library Upgrades + + - Jetty to 9.2.11 + +Platform 1.20 + +* TestingHttpClient + + We have deprecated the TestingHttpClient constructors that take a + Function in order to remove ambiguity when using + a lambda. Use the Processor variants instead. + + We have added a TestingHttpClient.setProcessor() method. + +* HttpClient + + We renamed the "http-client.read-timeout" configuration option to + "http-client.idle-timeout". + + We added the "http-client.request-timeout" configuration option. + + We deprecated the unused "http-client.keep-alive-interval" configuration + option. + +* HttpServer + + We added the "http-server.accept-queue-size" configuration option. + +* Threads + + Thread factories provided by the Threads utility class now add threads to a + ThreadGroup. Some tools organize threads by ThreadGroup which makes it + easier to work with servers with lots of threads. The ThreadGroup is named + using: + + String.format(nameFormat, "group") + + This means the nameFormat must expect a String argument instead of an int. + To ease migration, the code adapts formats with a single %d, but this may + be removed in a future release. + +* Discovery + + We have added an /admin/stop-announcing resource to the admin port. + When a PUT request, using Basic authentication with the username and + password configured as admin-server.username and admin-server.password, + is sent to this resource then the discovery announcer will revoke all + announcements and shut down. + + This change adds a dependency from JmxHttpModule to Discovery, which + might break TestServer. TestServer usually doesn't need JmxHttpModule, + so it can be removed. + +* Logging + + We have replaced use of SLF4J in the logging backend with a custom logging + implementation directly connected to Java util logging. This eases + management of logging dependencies as the core codebase no longer depends + directly on any third party logging frameworks. + +* Library Upgrades + - Guice to 4.0 + +Platform 1.19 + +* Java 8 is now required + +* JSON + + We added JSON serialization support for the JDK 8 Optional and JSR 310 + date/time classes. + +* Library Upgrades + - Jackson to 2.4.4 + +* Maven plugin upgrades + - findbugs-maven-plugin to 3.0.1 + - maven-dependency-plugin to 2.10 + +Platform 1.18 + +* Bootstrap + + We added a new @AcceptRequests annotation for methods that start accepting + requests into the application. The methods will be invoked after all + @PostConstruct methods have been invoked. Upon shutdown, any @PreDestroy + methods in the class will be invoked before those in non-@AcceptRequests + classes. + +* HttpServer + + We changed HttpServer.start() to be an @AcceptRequests method in order to + ensure all dependencies are initialized before accepting client requests. + + This adds a dependency from HttpServerModule to Bootstrap, which might break + unit tests that use HttpServerModule without Bootstrap. Such unit tests + should be modified to use Bootstrap--see the sample server's TestServer. + +* Reporting + + We now report upon startup a ReportCollector.ServerStart metric with a value + of 1. This may be used to track server restarts. + +* Library Upgrades + + - Jersey to 2.17 + - logback to 1.1.3 + +Platform 1.17 + +* Maven 3.2.3 required + + We increased the minimum version of Maven to 3.2.3. + +* Deprecated APIs removed + + - AsyncHttpClient + - BalancingAsyncHttpClient + +* Deprecations + + - HttpClientAsyncBindingBuilder has been deprecated. Use + HttpClientBindingBuilder instead. + - AsyncHttpClientModule has been deprecated. Use HttpClientModule instead. + +* Packaging + + maven-assembly-plugin is now configured to generate Posix tar files instead + of Gnu tar files. + +* [Bug] Report correct values for the HttpClient WrittenBytes metrics. + +* Library Upgrades + + - Jetty to 9.2.10 + +Platform 1.16 + +* RC4 disabled + + We disabled all RC4 ciphers in both HttpServer and JettyHttpClient. + +* Launcher + + The launcher jar now shades its jnr-posix and asm dependencies. + +* Maven plugin upgrades + - maven-assembly-plugin to 2.5.3 + +Platform 1.15 + +* Jetty security vulnerability fix + + Platform versions 1.07 through 1.14 have a critical information bleed + vulnerability in the Jetty server. + +* RelativeUriBuilder + + We created RelativeUriBuilder for building the relative URIs needed by + BalancingHttpClient. + +* Bootstrap + + We created Bootstrap.bootstrapApplication(Class, Function) + for setting the application name based on configuration. + +* We reimplemented SmileBodyGenerator to be a StaticBodyGenerator. + +* Library Upgrades + + - Jetty to 9.2.9 + +Platform 1.14 + +* JAX-RS + + We created JaxrsBinder.bindInjectionProvider() to support JAX-RS Custom + Injection Providers. Use this to bind a type to a Supplier of that type. + The objects returned from the Supplier will be field and method injected + by Jersey and any PostConstruct and PreDestroy methods will be called at + the appropriate time. + + We now enable the JAX-RS MIME Multipart support. + +Platform 1.13 + +* HttpClient + + Using a BodyGenerator that is not a StaticBodyGenerator with JettyHttpClient + when there are more concurrent requests than half the maximum threads in the + thread pool can lead to deadlocks. To mitigate this, we have made the + following changes: + + - We created a new marker interface BodySource. + - BodyGenerator extends BodySource. + - We created the interface DynamicBodySource to replace BodyGenerator. + - We deprecated BodyGenerator. Consumers should use BodySource instead. + Producers should recode to use DynamicBodySource. + - We deprecated StaticBodyGenerator.write(OutputStream). + StaticBodyGenerator is not deprecated, but after the deprecation period + it may extend BodySource instead of BodyGenerator. + - We deprecated Request.getBodyGenerator(). Use the newly created + Request.getBodySource() instead. + - We deprecated Request.Builder.setBodyGenerator(BodyGenerator). Use + Request.Builder.setBodySource(BodySource) instead. This change will + probably have a longer than usual deprecation period. + - We replaced SingleUseBodyGenerator with the LimitedRetryable interface. + - We created InputStreamBodySource for reading the body from an + InputStream. HttpClient implementations will not call close() on the + InputStream. + - We created BodySourceTester to help test BodySource implementations. + - We changed the reporting client to use DynamicBodySource. + +* BalancingHttpClient + + - The RetryingResponseHandler used by BalancingHttpClient will only include + an exception's stack in the log once per 30 seconds per exception class. + - If a Request has a BodySource that implements LimitedRetryable, the + BalancingHttpClient will not retry a request unless + LimitedRetryable.isRetryable() returns true. + +* Events + + - We changed the HttpEventClient to use a private IO pool. We recommend + that applications remove HttpEventModule from their Bootstrap invocation + unless they have a need to send events. + - We deprecated EventClient.post(EventGenerator) and EventGenerator. + This interface effectively required implementation on top of + BodyGenerator. No replacement is currently planned. + - We changed the HttpEventClient.post(Iterable) and .post(T...) + implementations to use DynamicBodySource. + +Platform 1.12 + +* We improved the previous BodyGenerator bug fix to remove a race condition. + +Platform 1.11 + +* SingleUseBodyGenerator + + We created SingleUseBodyGenerator, an abstract class implementing + BodyGenerator that enforces only being called once. The BalancingHttpClient + will not retry a request once a SingleUseBodyGenerator has been called. + +* Trace Token improvements + - TraceTokenManager.registerRequestToken() now returns a TraceTokenScope + which can be used with try-with-resources to restore the thread's previous + token state. + - TraceTokenManager methods are now all static. + - TraceTokenModule is no longer needed and has been deprecated. + - HttpClient now propagates the trace token to BodyGenerator and + ResponseHandler calls. + +* Library Upgrades + + - AutoValue to 1.0 + - Jetty to 9.2.7 + - jnr-posix to 3.0.9 + +* [Bug] Interrupt the BodyGenerator thread on write failure. This might have + been the cause of HttpClient hangs. + +* [Bug] Fixed status code for JAX-RS query parameter parse error + + When a query parameter cannot be parsed (for example in Integer parameter + whose supplied value could not be converted to an Integer), a + 400 BAD REQUEST will be issued instead of the Jersey default 404 NOT FOUND. + +Platform 1.10 + +* [Bug] NPE if config class bound both with and without a config prefix. + +* [Bug] DiscoveryBinder.bindDiscoveredHttpClient() exporting HttpClient + metrics with wrong name. + +Platform 1.09 + +* Deprecated APIs removed + + - Bootstrap public constructors + - ConfigAssertions.assertDefaults() + - ConfigAssertions.assertDeprecatedEquivalence() + - "node.binary-spec" and "node.config-spec" configuration options + - MeterStat + - TimedStat + - Duration.convertTo() + +* Guava 17 compatibility + + We reverted use of features preventing running with Guava 17. + +* Library upgrades + + - slf4j to 1.7.8 + +Platform 1.08 + +* Configuration module defaults + + Guice modules may now override default configuration values by implementing + ConfigurationDefaultingModule.getConfigurationDefaults(). Such defaults + override those in the config classes but are themselves overridden by + Bootstrap.withApplicationDefaults(). + +* AsyncSemaphore + + AsyncSemaphore is a non-blocking concurrency throttler for asynchronous + tasks. It guarantees that no more than a fixed number of asynchronous tasks + will be outstanding at any point in time without blocking any submission + threads. Task completion is determined by a ListenableFuture returned by the + supplied submitter function. + +* Discovery read timeout + + We reduced the discovery client's default read timeout to 5 seconds. This + is to avoid announcements timing out of Discovery when one or more + Discovery servers hang. + +* HttpClient request queue size default change + + We reduced the http-client.max-requests-queued-per-destination default to + 20. We recommend this be set to the same value as + http-client.max-connections-per-server as larger values can cause requests + to queue for inordinate amounts of time. + +* Stack traces in HTTP error responses + + The HTTP server now shows stack traces in error responses if + "http-server.show-stack-trace" is set to "true". + +* Node ID logged on startup + +* [Bug] HttpClient metric names now start with "HttpClient" instead of + "AsyncHttpClient". + +* [Bug] Fixed JettyHttpClient IllegalStateException when handling an exception + thrown from a BodyGenerator. + +* Library upgrades + + - Jetty to 9.2.6 + +Platform 1.07 + +* AbstractConfigurationAwareModule + + Extending this module makes it easy to access configuration at binding time, + which allows conditionally installing modules or bindings based on config. + +* Library upgrades + + - Jetty to 9.2.3 + - slf4j to 1.7.7 + - logback to 1.1.2 + - Jackson to 2.4.3 + - Jersey to 2.13 + - Joda-time to 2.5 + - Mockito to 1.10.8 + - cglib to 3.1 + +Platform 1.06 + +* SSLv3 disabled in both http-client and http-server + +* AutoValue support + + The libarary pom now specifies a version and excludes for Google AutoValue. + Google AutoValue simplifies + the creation of immutable value classes. + + The sample server has been updated to demonstrate AutoValue. + +* Smile support in http-client + + We added SmileBodyMapper, SmileResponseHandler, and + FullSmileResponseHandler classes to the http-client sub-project. + +* SetThreadName + + The SetThreadName class takes advantage of try-with-resources to safely rename a + thread and restore its original name when it goes out of scope. + + Here's a usage example: + + try (SetThreadName ignored = new SetThreadName("query-%s", queryId)) { + ... + } + +* Rack moved to separate project + + We moved the rack packaging support into the platform-rack project. + +* We changed the Logger API to work directly against Java's built-in logging APIs. + +* HttpClient + + - Name and group Jetty HTTP client threads + - Always use daemon threads in HTTP client + - Add getResponseBytes() and getResponseBody() to FullJsonResponseHandler + +* We added Announcer.forceAnnounce() to immediately send a discovery announcement. + +* We added more utility methods to Closeables + +* We changed Duration.nanosSince() to convert to the most succinct unit. + +* Library upgrades + - Guava to 18.0 + +Platform 1.05 + +* Findbugs run by default + + Findbugs is now hooked into the verify lifecycle phase. (The previous + attempt was missing a step.) Failing the build on Findbugs failures is + now disabled by default. + +* Reporting + + We added support for reporting string values to KairosDB. Reported values + that are not Number or Boolean have their toString() reported as a string. + +* MaxDataSize and MinDataSize + + We added MaxDataSize and MinDataSize Bean validations. + +* HttpClient + + We added a "http-client.max-requests-queued-per-destination" config option + (default 1024). + +* Thread Factory + + We have changed the thread factory implementations to capture the thread + context class loader during factory creation and set this class loader on + threads created by the factory. + +* Smile mapper + + We have added a JAX-RS mapper encoding responses using Smile. Smile is a + highly efficient binary encoding of JSON data. For more information on + Smile see: http://wiki.fasterxml.com/SmileFormatSpec + +* Library upgrades + - joda-time to 2.4 + - Jersey to 2.12 + - Jetty to 9.2.2 + - Jackson to 2.4.2 + +* Maven plugin upgrades + - findbugs-maven-plugin to 3.0.0 + +* [Bug] Don't unannounce from discovery if Announcer never started. + +Platform 1.04 + +* RPM packaging + + We added support to rest-server-base for packaging projects into RPMs. + This profile is activated on Linux by creating a file .build-rpm in + the root of the module or submodule. This file can be empty. The RPM + is attached as an additional artifact. + + The username used to run the server defaults to ${project.artifactId} + and can be changed by setting the property ${project.rpm.username}. + + The RPM will stop the server upon upgrade or removal and will refuse + to install if the server username does not exist. + +* Maven 3.2.2 required + + In order to support RPM packaging, we increased the minimum version of + Maven to 3.2.2. + +* Library upgrades + - Guava to 17.0 + +* Maven plugin upgrades + - findbugs-maven-plugin to 2.5.2 + - maven-release-plugin to 2.5 + +Platform 1.03 + +* JAX-RS 2 + + We have upgraded to Jersey 2.9.1 which implements the JAX-RS 2 specification. + Platform clients need to change the dependency: + + com.sun.jersey + jersey-core + + to: + + javax.ws.rs + javax.ws.rs-api + + As part of this upgrade we have introduced an explicit binder for JAX-RS + resources. For example, the following registers the person JAX-RS resource: + + JaxrsBinder.jaxrsBinder(binder).bind(PersonResource.class) + + The new explicit binding code is backwards compatible with the "binding + scraping" code, so you do not need to immediately update your code. The new + JaxrsModule will print a warning for each resource that was not explicitly + bound, so it is easy to find update existing code. Once you have updated + everything, use JaxrsModule.explicitJaxrsModule() to prevent regressions. + In a future version we will remove the backwards compatibility, so please + update soon. + + We have removed JaxrsBinder.bindResourceFilterFactory(). Users of this will + have to convert their ResouceFilterFactory implementations to JAX-RS 2 + server filters. + +Platform 1.02 + +* BalancingHttpClient + + We added DiscoveryBinder.bindDiscoveredHttpClient(String) and + DiscoveryBinder.bindDiscoveredHttpClient(String, ServiceType) methods which + bind the HttpClient with the @ServiceType(type) annotation. + +* Library updates + - jacoco to 0.7.1 (fixes failures when compiling with JDK 8) + +Platform 1.01 + +* Change packaging to not include zip file dependencies + +* Library updates + - jmxutils to 1.18 + - findbugs-annotations to 2.0.3 + +* [Bug] Fix Discovery announcer client to handle zero announcements + +* [Bug] Fix inadvertent masking of errors when detecting jmx agent + +Platform 1.00 + +* HttpServer + + We restored support for compressed request bodies. This had been removed in + the Jetty 9 upgrade in Platform 0.91. + +* JsonCodec + + We added JsonCodec.withoutPretty() which returns a JsonCodec that doesn't + add whitespace to the encoded JSON. + +* Assertions + + We added Assertions.assertNotContains() methods. + +Platform 0.99 + +* HTTPClient + + - We have disabled following HTTP redirects by default. Redirects may be + enabled per-request using Request.Builder.setFollowRedirects(true). + +* HttpServiceSelector + + HttpServiceSelector is now deprecated. Code using it should be rewritten + to use a BalancingHttpClient instead. + +Platform 0.98 + +* JaCoCo replaces Cobertura + + We replaced Cobertura with JaCoCo. JaCoCo is now run by default in projects + that use library or rest-server-base as a parent POM. + +* TimeStat and SparseTimeStat + + - TimeStat and SparseTimeStat now export a Total metric. + + - We added a BlockTimer.timeTo() method for changing the stat object used + to record the time. + +* BoundedExecutor + + We added BoundedExecutor, which guarantees that no more than maxThreads will + be used to execute tasks. + +Platform 0.97 + +* Events + + - The event client now retries failed requests. + - The Future returned from EventClient.post() now fails with + UnexpectedResponseException when the collector service returns a failure + response code. + +Platform 0.96 + +* JMX + + - We no longer announce the "jmx" service to discovery. + + - We no longer enable remote JMX by default. Remote JMX may be enabled with + the "jmx.enabled" config option. + +* HttpClient + + This release fixes a couple of performance issues and bugs in the HttpClient: + - The client was inadvertently requesting GZIP responses. We've disabled this + behavior since it affects CPU utilization. + - We've fixed an issue in the way the response buffers are managed that + affects performance due to unnecessary resizing. + - The maximum response size was hard-coded instead of using the existing + config option. + +* Library upgrade + - Jetty to 9.1.4 + +Platform 0.95 + +* Reporting + + - We changed some metric names: + + - RequestStats.* renamed to HttpServer.* + - DetailedRequestStats.RequestTime.* renamed to HttpServer.RequestTimeByCode.* + - Failure.Count renamed to ServiceClient.Failure.Count + - ResponseTime.* renamed to ServiceClient.RequestTime.* + + - We no longer report a MaxError metric from TimeStat and DistributionStat. + + - We no longer include a "package" tag in reported metrics. + + - We removed the HttpClient._BindingAnnotation_.Count metric as redundant. + +* Stats + + We have deprecated CounterStat.update(long), replacing it by CounterStat.add(long) + + We created SparseCounterStat, etc. for use with report collections. These stats + objects do not report any metrics for minutes in which no data were added to the + object. They also do not export any attributes to JMX. + +* Configuration + + The @ConfigMap annotation is no longer necessary and is deprecated. + +Platform 0.94 + +* The launcher now invokes the JVM with the following default switches: + + -server + -XX:+UseConcMarkSweepGC + -XX:+ExplicitGCInvokesConcurrent + -XX:+HeapDumpOnOutOfMemoryError + -XX:+AggressiveOpts + -XX:+DoEscapeAnalysis + -XX:+UseCompressedOops + -XX:OnOutOfMemoryError=kill -9 %p + +* We removed unnecessary features from the Rack launcher: + + - The rack launcher no longer copies the etc directory to rack/config + - The rack config path is now always "rack/config.ru" + - The rack environment is now always "production" + +Platform 0.93 + +* Reverted the Hamcrest support in JsonTester.assertJsonEncode() + +* Library upgrade + - Jetty to 9.1.3 + +Platform 0.92 + +* We created TestingReportCollectionFactory + +* JsonTester.assertJsonEncode() now supports Hamcrest matchers + +Platform 0.91 + +* AsyncHttpClient collapsed into HttpClient + + - We moved the executeAsync() method into the HttpClient interface. + - The AsyncHttpClient interface is now deprecated. + - The AsyncHttpClient.AsyncHttpResponseFuture interface is now named + HttpClient.HttpResponseFuture + +* HttpClient uses Jetty + + We replaced the Apache and Netty HttpClient implementations with + JettyHttpClient, a Jetty implementation. + +* Configuration warnings are no longer sent to the event service upon startup. + +* We added a Bootstrap.quiet() method to disable logging of configuration. + +* We tightened the type bounds of Assertions.assertInstanceOf() to catch some + errors at compile time. + +* Library upgrades + - Jetty to 9.1.2 + - JRuby to 1.7.11 + +* Maven plugin upgrades + - maven-duplicate-finder-plugin 1.0.6 + +Platform 0.90 + +* JsonCodec + + The JsonCodec.jsonCodec() factory method now uses Guava's TypeToken rather than + Guice's TypeLiteral. This backwards incompatible change allows using the json + module without a dependency on Guice. + +* TestingHttpServer now supports resources + +* Library upgrades + - Guava to 16.0.1 + - logback to 1.1.1 + - findbugs-annotations to 2.0.2 + - testng to 6.8.7 + +Platform 0.89 + +* [Bug] BalancingHttpClient no longer dequotes and requotes the URI. + +* [Bug] HttpUriBuilder now quotes the '+' character. + +Platform 0.88 + +* Bootstrap and Logging + + - Bootstrap now initializes logging before the withModules() call. Any + test code that needs to call doNotInitializeLogging() now must do + so before the withModules() call. + + - We modified the archetype Main classes to place the Bootstrap call inside + the try block. This allows more startup exceptions to be logged. We + recommend applications make a corresponding change to their Main class. + +* We removed the deprecated Bootstrap.strictConfig() and + ConfigurationFactory.getUsedProperties() methods. + +* BalancingHttpClient now avoids concurrent calls to the same instance and + uses discovery changes that happen after the first attempt. + +* [Bug] BalancingHttpClient instances couldn't be configured. + +* [Bug] JAX-RS response time reporting threw exceptions when a resource class + had multiple methods with the same name. + +* [Bug] The launcher wasn't disconnecting from the terminal upon server start. + +Platform 0.87 + +* The node.pool configuration property now allows uppercase in pool names. + +* Reporting + + - JAX-RS now reports response times per resource method and response code. + JaxrsModule now depends on ReportingModule. + + - Permute reported metric names as necessary to make them acceptable to KairosDB + +Platform 0.86 + +* Reporting + + - The reporting client now uses discovery. The "report.uri" configuration + property has been removed. + + - Reporting is now enabled by default. It may be disabled with the + "reporting.enabled" configuration property. + + - The "report.tags" configuration property has been renamed to + "reporting.tags". + +Platform 0.85 + +* [Bug] BalancingHttpClient preserve the query part of the URI + +Platform 0.84 + +* Reporting (experimental) + + - The values of any "type" and "name" tags are used to prefix the metric + name. + + - The http-client RequestStats are exposed to reporting. + +* We added Threads.threadsNamed(String) and Threads.daemonThreadsNamed(String) + methods to simplify creating a ThreadFactory. + +* We added decayed total to Distribution + +* We added merge support to CounterStat and DecayCounter + +* [Bug] Ignore IOException in Closeables.closeQuietly() + +* Library upgrades + - validation-api to 1.1.0.Final + - logback to 1.0.13 + +Platform 0.83 + +* Application name + + We've added a static factory for Bootstrap which takes the name of the + application server: + + Bootstrap.bootstrapApplication("name-of-application") + .withModules(...) + + The previous Bootstrap constructors are deprecated. + + The application name can be obtained from NodeInfo.getApplication(). + + NodeModule and TestingNodeModule have a dependency on the newly created + ApplicationNameModule. This dependency is supplied by Bootstrap. + + The application name is included in a tag named "application" in reported + metrics. + +* HttpServiceBalancer metrics + + HttpServiceBalancer now reports: + - client response times by success/failure and target URI + - failures by failure category and target URI + + DiscoveryModule now has a dependency on ReportingModule. + +* TestingTicker + + TestingTicker now has public visibility in the testing package. It is an + implementation of Ticker that can be manually incremented. + +* Library upgrades + - Guava to 15.0 + +Platform 0.82 + +* We added JaxrsBinder.bindResourceFilterFactory for adding custom + ResourceFilterFactory instances to the HTTP server. + +* The HTTP server no longer sends HttpRequest events. + +Platform 0.81 + +* We made JAX-RS resource methods Bean-validate arguments of List and Map. + +* JMX agent + + We've replaced the custom JMX agent in Platform by the one built into the + JVM. This change makes it possible to use the VisualGC plugin for VisualVM + when jstatd is also running in the same machine as the Platform-based + process. + + As a result of this change, we've also removed the 'jmx.rmiserver.hostname' + configuration option in favor of Java's native configuration option + 'java.rmi.server.hostname'. + +* We demoted several legacy configuration properties to defunct. We renamed + 'discovery.uri' to 'testing.discovery.uri' to make it obvious that it should + never be used in production. + +* We added an experimental management wrapper for a thread pool executor. + +* We added the ConfigurationAwareProvider interface to implementing custom + configuration providers. + +* We added byte array methods for JsonCodec. + +* We added 1,5,10, and 25 percentiles to Distribution and added manual + percentile specification to Distribution getPercentiles + +Platform 0.80 + +* HttpServer reporting + + We instrumented HttpServer to use the reporting module. HttpServerModule() + now depends on ReportingModule(). + + We added stats broken down by response code and removed a redundant + CounterStat. + +* Units + + We've made a backwards incompatible change to the Duration API to align with + the DataSize API. The new Duration code has many new features: + + - Duration remembers the original unit, so parses and prints round trips. + - toMillis() returns a long so it is easier to use with normal Java APIs. + - roundTo(TimeUnit) return a long in the specified unit. + - convertToMostSuccinctTimeUnit() will select the TimeUnit to produce + the easiest to read value. + - timeUnitToString(TimeUnit) to get a clean short name for TimeUnits. + + To update to the new api, you will need to: + + toMillis() now returns a long instead of a double + - Use getValue(MILLISECONDS) for the old behavior. + + toString() returns a string in the original units instead of millis + - Use toString(MILLISECONDS) for the old behavior. + + convertTo(TimeUnit) has been deprecated and will have an incompatible + return value in the future. + - Use getValue(TimeUnit) for the old behavior. + - Use roundTo(TimeUnit) to get a long in the specified unit. + +* We added constructors to TestingHttpClient allowing instances to send + checked exceptions. + +* Reporting (experimental) + + We made refinements to the experimental reporting module. + + - We split ReportingModule in two: ReportingModule is needed in order to + instrument code and ReportingClientModule is needed to send the data to + the time series database. + + - reportBinder.bindReportCollection() now requires an additional method call + to specify the base ObjectName to use in exporting. Either + .withGeneratedName() or as(String) can be used. + + reportBinder.bindReportCollection(StoreStat.class).withGeneratedName(); + +* [Bug] Fixed typo breaking Bootstrap.withApplicationDefaults(). + +* [Bug] JsonTester now uses correctly configured codecs. + +Platform 0.79 + +* HttpClient exception handling + + We have redesigned exception handling in the HttpClient. In the new design, + the handle or handleException method of the Response handler should be called + once per execution. To make this change we had to make some backwards + incompatible changes. + + The AsyncHttpClient.executeAsync method now returns a ListenableFuture instead + of a CheckedFuture. Any caller that was using the checkedGet() method of + CheckedFuture will need to change to get() and handle the exceptions or use + one of the helper methods in Guava Futures. + +* Reporting (experimental) + + We added an experimental API for reporting metrics into a KairosDB + time-series database. The API and its configuration options are subject + to change. + + - Collecting data + + The @Gauge annotation may be placed on a getter in order to cause the + attribute to be both reported into the database and exported to JMX. If + the attribute is not to be exported to JMX, the @Reported annotation may + be used instead. The getter must return a number or boolean. + + @Nested and @Flatten also work for reporting. + + CounterStat, DistributionStat, and TimeStat have been extended to support + reporting. + + The newly created Bucketed abstract class may be extended in order to + implement custom stats objects that support reporting in time buckets. + + - Exporting report objects + + The ReportBinder.export() methods behave like ExportBinder.export() except + they cause the bound objects to be exported to both reporting and JMX. + + The ReportExporter may be used to dynamically export and unexport objects + to just reporting. + + - Report collections + + The ReportBinder.bindReportCollection() method may be used to break down + metrics by one or more keys. For example: + + public interface StoreStat + { + CounterStat added(@Key("size") int size, + @Key("successful") boolean isSuccessful); + }; + + reportBinder.bindReportCollection(StoreStat.class); + + will bind an implementation of StoreStat. A call to added(10, false) will + return a CounterStat bound to both reporting and JMX with the name + "type=StoreStat,name=added,size=10,successful=false". + + - Reporting client + + ReportingModule enables reporting of collected data to the + time-series database. Configuration parameters are: + + report.uri - The URI of the KairosDB database + + report.tag - A table of additional tag/value pairs to include in all + reported data. For example, report.tag.foo=bar will include + the additional tag foo=bar + + The sample server has been updated to support reporting. + +* Library upgrades + - jmxutils to 1.14 + +* [Bug] Futures returned from TestingHttpClient.executeAsync() didn't invoke callbacks. + +Platform 0.78 + +* Rack Packaging + + - Removed generation of gemfile.jar in favour of using Bundler’s standalone. + - Changed tar generation to include specific files instead of by exclusion to prevent pulling in unnecessary + project files. + - Incorporated asset precompilation fix + - Deprecated use of the bundler maven plugin as it didn’t work correctly or log useful information when it + failed. May want to wrap the above changes in a plugin in the future (currently using command lines via Ant + plugin). + +* Rack + + - Added intiialization code to ensure Rails application correctly loads gems from the standalone gem bundle. + - Fixed an issue in ServletAdapter that assumed status code was always an integer and caused exceptions when + it was actually a string + - Fixed issue that prevented Rails apps from setting more than one cookie at a time. + - Re-implemented hack to override logging as a Railtie so it could be correctly scheduled in the application + initialization sequence. All Rails apps will now correctly log everything (including startup logging) to + launcher.log. + - Updated/added gems to the test resources to ensure unit tests run correctly (which accounts for most of the + file changes) + +Platform 0.77 + +* HttpClient + + - The signature of ResponseHandler.handleException() has changed to: + + T handleException(Request request, Exception exception) + throws E; + + This permits the method to return a default value. + + - We added a new DefaultingJsonResponseHandler which returns a default + response upon any error. + +* Discovery client service pool default + + Service pools now default to the configured value of 'node.pool'. + +* SerialScheduledExecutorService + + We added SerialScheduledExecutorService to the testing module. It is a + test utility implementation of ScheduledExecutorService. See + https://github.com/markkent/serial-executor-service for more information. + +* Stats + + - MeterStat has been deprecated. DistributionStat should be used insead. + + - TimerStat has been deprecated. The newly created TimeStat should be used + instead. + + - The Http client and sever RequestStats have been changed to use + DistributionStat and TimeStat. + +* TestingHttpServer now injects a TraceTokenManager, so trace tokens are + available to other injected modules. + +* Library upgrades + - Netty to 3.6.6.Final + +* Maven plugin upgrades + - build-helper-maven-plugin 1.8 + - maven-assembly-plugin 2.4 + +* [Bug] Fixed discovery client's inability to refresh services + +* [Bug] Fixed missing module in Rack server. + +Platform 0.76 + +* [Bug] Fixed Guice error when binding AsyncHttpClients both normally and privately. + +* [Bug] Fixed the Netty http client's handling of exceptions. + +Platform 0.75 + +* Experimental modules + + We removed the trailing "-experimental" from all artifact names. + Any APIs that are experimental should be annotated with @Beta instead. + +* We moved the /v1/jmx/mbean resource to /admin/jmx/mbean on the admin port. + +* We removed the jmx-http-rpc-experimental artifact and the JmxHttpRpcModule + it defined. + +* We moved Announcer and ServiceAnnouncement into the + com.proofpoint.discovery.client.announce package. + +* Discovery aware balancing HttpClient + + We added a BalancingHttpClient implementation which takes relative URLs, + implements discovery lookup, and retries requests. + + To use, bind to a discovery service type: + + DiscoveryBinder.bindDiscoveredHttpClient("storage", StorageClient.class) + .withFilter(SomeFilter.class) + .withTracing(); + + The resulting HttpClient can be injected in the normal way: + + @Inject + StorageUser(@StorageClient HttpClient httpClient) + + The resulting HttpClient differs from a classic HttpClient in that it that + the Request URI must be relative: + + httpClient.execute(prepareGet() + .setUri(URI.create("v1/content/foo")) + .build(), + new StorageResponseHandler()); + + The BalancingHttpClient will send the request to some base URI obtained from + the discovery service for the ServiceType that the HttpClient was bound to + and for the pool obtained from configuration. If there are no available + instances, it will throw a ServiceUnavailableException. + + The BalancingHttpClient will retry the request, up to a configurable number + of total attempts, if the underlying HttpClient reports either an exception + through ResponseHandler.handleException() or a response with a status code + of 408, 500, 502, 503, or 504. The server can suppress retries by including + an "X-Proofpoint-Retry: no" header in its response. + + The ResponseHandler given to the BalancingHttpClient will be called at most + once. The BalancingHttpClient won't retry if the caller's + ResponseHandler.handle() method throws an exception. + + The Request and its BodyGenerator may be required to produce their contents + many times. + + Alternatively, use DiscoveryBinder.bindDiscoveredAsyncHttpClient() to bind + an AsyncHttpClient. + +* We removed support for JSONP and the HTML form of /v1/jmx/mbeans. + +* We explicitly specified UTF-8 as the charset for configuration property + files. + +* We renamed ConfigAssertions.assertDeprecatedEquivalence() to + assertLegacyEquivalence(). + +* We added TestingHttpClient to simplify testing. + +* QuantileDigest + + We made a number of improvements to QuantileDigest. QuantileDigests now support + negative values, can be serialized/deserialized and merged with other QuantileDigests. + + Important: QuantileDigest is no longer thread-safe, so if you have code that relies + on this guarantee, you'll need to update it to provide thread-safety at that layer. + +* We added a method to TimedStat to time a block using try-with-resources: + + try (BlockTimer ignored = timedStat.time()) { + ... + } + +* We added an addDiscoveredService(String type, String pool, String uri) + method to InMemoryDiscoveryClient. + +* We changed the discovery client to no longer prefer https instances over + http instances, but to only use https for any instance that supports both + schemes. + +* We removed support for loading the service inventory over http or https. + +* [Bug] Fixed handling of missing content type for JSON http responses. + +* We deprecated ConfigAssertions.assertDefaults(). + +* We upgraded Jackson to 2.2.2 and disabled property mutator inference. + +Platform 0.74 + +* The new launcher status command now reports a running instance of a server + started by the old launcher + +Platform 0.73 + +* Applications can now use Bootstrap.withApplicationDefaults() to specify + default configuration values. + +* Upon startup, the main thread will no longer wait for the initial discovery + announcement. + +* Launcher fixes + + - Many jvm.config files were depending on the old launcher's interpreting + spaces and other shell metacharacters. The launcher will now print an + error message and refuse to start if the jvm.config file contains spaces + or shell quotes, unless the jvm.config file contains the comment + "# allow spaces". + + - The new launcher can now shut down a server started by the old launcher. + + - [Bug] The new launcher can now more reliably find the installation directory. + +* [Bug] Don't create javadoc on non-release builds. + +* Maven plugin upgrades + + - maven-resources-plugin 2.6 + - maven-source-plugin 2.2.1 + - maven-surefire-plugin 2.14 + +Platform 0.72 + +Temporarily allow event names to start with lowercase letters, etc. + +Platform 0.71 + +* Log rotation changes + +The HTTP request log is now uses the same log rotation semantics as the server +log. The request log rotation is configured using the new +"http-server.log.max-size" and "http-server.log.max-history" configuration +options. The old "http-server.log.retention-time" configuration option is +deprecated and does nothing. + +The "log.max-size" configuration option takes a DataSize and replaces the +now-legacy "log.max-size-in-bytes" option. + +* Launcher reimplemented in Java + +The REST server launcher has been rewritten in Java. Changes include: + + - Fixes numerous race conditions. + - Implements the force-reload command required by the specification the + launcher allegedly implements. + - Removes dependency on Ruby. + - The launcher waits for the server to finish initialization (return from + main method) and returns an error status code if initialization fails. + - The launcher will no longer hang if the server fails to stop. + +* BETA: Bootstrap for testing + +The Bootstrap class performs may initialization tasks for servers, but when +testing not all installation tasks are desired. Specifically, the Bootstrap +class loads configuration properties from a file specified using System +property. The new setConfigurationProperty and setConfigurationProperties +have been added to supply the properties directly. Another, problematic +task is logging initialization, which can now be disabled with the new +doNotInitializeLogging() method. For an example of a Bootstrap test see +com.proofpoint.platform.sample.TestServer. + +* ValidationAssertions.assertValidates() now returns its argument. + +* JsonTester + +We created a JsonTester utility class for unit testing JSON encoders and +decoders. JsonTester.assertJsonEncode() asserts that a given object encodes +to an expected JSON strucure. JsonTester.decodeJson returns the result of +decoding a given JSON structure with a given codec. In either case, the +JSON structure is specified as an Object, which can be a String, an Integer, +a Double, a Boolean, null, a List of JSON structures, or a Map of String to +JSON structures. + +See TestPersonWithSelf and TestPersonRepresentation in the sample server for +examples of how to use JsonTester. + +* Configuration improvements + + - The ConfigurationFactory(Map properties) constructor + (typically used in unit tests) is now strict, requiring all supplied + properties to be used. + - Configuration bound inside private binders now works correctly. + +* Removal of jax-rs classes from clients + +We have replaced all uses of jax-rs classes in clients with classes from Guava +or new classes in the Platform http client. In particular, the HttpHeaders and +MediaType classes in jax-rs has been replaced with HttpHeaders and MediaType +in Guava, and the Status and CacheControl classes have been replaced with +HttpStatus and CacheControl in the Platform http-client. This reduces the size +of clients and avoids some packing problems with the latest jax-rs releases. +Any existing code can continue to use the jax-rs versions of these classes, but +jax-rs is no longer required. + +* AsyncHttpClient fixes + + - This release contains a number of fixes for the AsyncHttpClient and additional + changes to ease debugging of async requests. In some cases the ResponseHandler + would not be notified of errors with the handleException method, even though + the future returned from the executeAsync method would be notified of the + error. To help with debugging, the executeAsync method now returns an + AsyncHttpResponseFuture which has a new method getState that can be used to + determine the current state of the http request. This is helpful when requests + seem to get lost after execution. Finally, we have changed the Netty setup to + properly name the worker threads, so you can more easily find the http client + threads. + + - AsyncHttpClient (Netty based) now shares I/O pools by default. To get a + private IO pool, use withPrivateIoThreadPool() when binding the client. + The pools running user code are still per-client. + + - AsyncHttpClient.executeAsync() no longer throws E. + +* Binding of HTTP static content + +We added HttpServerBinder, which binds static resources to be served by the +http server. + +* Temporarily allow the characters .:=,- in event names once again. + +* Add "X-Content-Type-Options: nosniff" header to all json responses to prevent + broken browsers from rendering the json as html, which is a potential XSS + vulnerability + +* Dependency changes + +com.google.code.findbugs:jsr305 was replaced with +com.google.code.findbugs:annotations + +The jsr305 jar contains only the JSR 305 annotations while the +annotations jar also contains additional, Findbugs specific +annotations. + +org.eclipse.jetty.orbit:javax.servlet was replaced with +javax.servlet:javax.servlet-api + +* Library upgrades + + - Jetty 8.1.10 + - Guava 14.0.1 + - slf4j 1.7.5 + - logback 1.0.11 + - Jersey 1.17.1 + - TestNG 6.8.1 + +* Internal library upgrades + + - Mockito 1.9.5 + - Hamcrest 1.3 + - jcommander 1.30 + - mysql-connector-java 5.1.24 + - h2 1.3.171 + - Apache httpclient 4.2.4 + - Apache httpcore 4.2.4 + +* Maven plugin upgrades + + - maven-compiler-plugin 3.0 + - maven-deploy-plugin 2.7 + - maven-duplicate-finder-plugin 1.0.4 + - maven-enforcer-plugin 1.2 + - maven-install-plugin 2.4 + - maven-jar-plugin 2.4 + - maven-javadoc-plugin 2.9 + +Platform 0.70 + +* Upgrade Jackson to 2.1.3 + +Jackson 1.x has been upgraded to 2.1.x. This requires substantial code and +dependency changes as all the artifacts are named differently and all the +classes are in different packages. The upgrade guide has more details: +http://wiki.fasterxml.com/JacksonUpgradeFrom19To20. + +The Json module now also supports Jackson Modules (more details at +http://wiki.fasterxml.com/JacksonFeatureModules). + +* True async http client + +The AsyncHttpClient has been rewritten on top of Netty. As part of this +work, the AsyncHttpClient has been converted into an interface and the +main implementation is NettyAsyncHttpClient. Additionally, the execute +method has been renamed to executeAsync so the AsyncHttpClient can extend +HttpClient. The change is to simply replace: + + asyncHttpClient.execute(request, responseHandler); + +with: + + asyncHttpClient.executeAsync(request, responseHandler) + +The other main change is when using the HttpClientBinder. The bindAsyncHttpClient +method now binds both an AsyncHttpClient and a HttpClient, so if you were binding +both before, you will need to remove the call to bindHttpClient. + +This API remains in BETA, and is subject to additional backwards incompatible +changes. + +* Socks proxy support in http client + +The new NettyAsyncHttpClient supports connecting to servers through a socks proxy. +This is particularly useful for accessing production servers. The easiest way +to use this feature is to establish a socks proxy using ssh, as follows: + + ssh -N -D 1080 any.server.running.sshd + +Then simply set the HttpClientConfig SocksProxy property either directly in code +or via the following config option: + + myclient.http-client.socks-proxy = localhost:1080 + +* Pretty-printed JSON + +The injected JsonCodecFactory and injected codecs no longer produce +pretty-printed JSON. The previous behavior was a bug. JsonCodecFactory +has a method to return a factory that produces pretty-print codecs. + +* Trace token improvements + + - The trace token (and other MDC items) is now included in the server log. + - The trace token is now included in events. + +* Configuration improvements + + - The error message that Configuration issues upon Bean validation failure + now identifies the attribute's configuration property, if any. + + - Configuration warnings are now sent to the event service upon startup + and once a day thereafter. + +* Disabled auto-detection of constructors and use of getters as setters in + Jackson. + +* The HttpClient and AsyncHttpClients now implement Closeable and properly + shutdown when used as lifecycle objects. Calling close shuts down the + connection pool for the underlying Netty HTTP client. + +* DataSize and Duration can now be serialized as json + +* Library upgrades + + - Upgrade jmxutils to 1.13 + - Upgrade jruby to 1.7.3 + +* H2 improvements + - Improve validation of H2EmbeddedDataSource config + - Allow use of H2 without an init script + - Enable mvcc support for embedded H2 + +* Removed deprecated features + - Support for v1 events has been removed. + - The EquivalenceTester static check methods have been removed. + +* [Bug] HTTPS servers are now announced in discovery using the hostname, not an IP address + +* [Bug] Don't throw an exception from http request logger when wall clock time goes backwards. + +Platform 0.69 + +* Library Upgrades + +In this release we have upgraded the following dependencies: + + - Guava 13.0.1 + - jetty 8.1.8 + - httpclient 4.2.2 + - bval-jsr303 0.5 + - joda-time 2.1 + +NOTE: Guava 13 removes transitivity of the JSR-305 artifact (commonly used for @Nullable). +If your project depended on this, you need to add the following to your project's pom.xml file: + + + com.google.code.findbugs + jsr305 + + +* Mappable exception for JSON decoding errors + +When there is a JSON decoding error, the JsonMapper.readFrom() method now throws newly defined +exceptions (instead of WebApplicationException). This allows them to be handled by an ExceptionMapper. +The newly defined exceptions are BeanValidationException and JsonMapperParsingException. + +* Workaround for InetAddress.getLocalHost() throwing UnrecognizedHostException in Java 7u5 on MacOS + + - The jmx module now gets the default value of the local IP address from the node module. + - The event-client module tries harder to get the local hostname + +* We now detect some incorrect usage to EquivalenceTester. + +To detect calls intended for EquivalenceTester.addEquivalentGroup(Iterable) that instead +call EquivalenceTester.addEquivalentGroup(T, T...), the latter method now prohibits calls +with a single argument that implements Iterable. + +* Configuration classes, getters, setters, and default constructors no longer need to be public. + +* @ConfigSecuritySensitive now redacts the configuration value in any generated error. + +* [Bug] @ConfigMap attributes can now be tested by ConfigAssertions.assertFullMapping() and + ConfigAssertions.assertDeprecatedEquivalence() + +* [Bug] Don't throw exception from StatsRecordingHandler if the system clock goes backwards. + +* We have removed the Cassandra module + +Platform 0.67 + +* No thread pool for CounterStat + +We changed the CounterStat implementation to use forward decay, which eliminates +the need for a thread pool (and having to manage a thread pool). + +* HttpUriBuilder a fluent api for HTTP URIs + +Building HTTP URIs using java.net.URI is difficult and error prone, so we added +a simple fluent API for building up HTTP URIs. For example: + + URI uri = uriBuilder() + .scheme("http") + .host("www.example.com") + .port(8081) + .replacePath("/a/b/c") + .replaceParameter("k", "1") + .build(); + +Additionally, the API can manipulate existing URIs: + + URI uri = HttpUriBuilder.uriBuilderFrom(URI.create("http://www.example.com:8081/?a=apple&b=banana")) + .replaceParameter("a", "apricot") + .appendPath("a/b/c") + .build(); + +* Bind Jackson key serializers and deserializers in Guice + +We've added the ability to bind Jackson key serializers and deserializers with +the JsonBinder. + +* Configuration types can now use a fromString method for coercion + +The configuration system will look for methods in the following order: + + fromString(String); + valueOf(String); + (String); + +* Quantile Digest + +We've introduced an implementation of Quantile Digests, a data structure for computing approximate +quantile statistics and histograms in sub-linear space with guaranteed error bounds, from the paper +"Medians and Beyond: New Aggregation Techniques for Sensor Networks". It supports +exponentially-decayed counts and can be used to query quantiles and histograms very efficiently. + +* DistributionStat + +We've deprecated MeterStat in favor of DistributionStat. This new implementation is based on +QuantileDigest and exposes statistically valid percentile metrics at various resolutions +(1/5/15-minute, all time). It's designed to be exported via jmxutil's @Managed annotations. + +* @ConfigSecuritySensitive + +We created the @ConfigSecuritySensitive annotation for configuration settings such as passwords. +When placed on a @Config setter, prevents the value of the setting from being logged upon startup. + +* @ConfigMap + +We added support for configuration maps. For example, given the setter: + + @Config("setting") + @ConfigMap(key = Integer.class, value = Double.class) + public void setSetting(Map value) + +the configuration settings: + + setting.1=0.059 + setting.5=2.543 + +will cause the setter to be called with ImmutableMap.of(1, 0.059, 5, 2.543). + +The "key" and "value" arguments both default to String.class and can be any type that the configuration +system can coerce. Furthermore, the "value" argument can be any configuration class. For example, +given a EndpointConfig class with setters annotated with @Config("hostname") and @Config("port"): + + @Config("endpoint") + @ConfigMap(EndpointConfig.class) + public void setFruit(Map value) + +the configuration settings: + + endpoint.foo.hostname=foo.example.com + endpoint.foo.port=8080 + endpoint.bar.hostname=bar.example.com + +will cause the setter to be called with a map with two entries. One entry will have key "foo" and +value an EndpointConfig with host "foo.example.com" and port "8080". The other entry will have key +"bar" and value an EndpointConfig with host "bar.example.com" and a default port. + +* Additional error checks in configuration + + - All configuration properties read from a configuration file must be used--Bootstrap.strictConfig() + now does nothing and is deprecated. + - Duplicate configuration property keys in a given configuration file now cause an error. + - Specifying a given configuration setting multiple times (through legacy properties) is now an error, + even if all the properties use the same string value. + +* We removed the restriction of entity bodies to PUT/POST requests that was added in Platform 0.65 +* We deprecated unused configuration settings in HTTPEventClient +* We limited the stats to log dependency to test scope + +Platform 0.66 + +* Java 7 is now required + +Platform 0.65 + +* Improvements in unit tests and testing utilities +* Modifications to allow building on Windows +* Restrict entity bodies to PUT/POST requests in HttpClient +* [Bug] Throw correct exception from HttpClient + +Platform 0.64 + +* Fix bug that prevented skeleton server from starting + +* Enforce use of HttpClientBinder + +The previous way of using HttpClient was to create a new instance of the +HttpClientModule or AsyncHttpClientModule for each client. The recent addition +of filter support added a binding API, HttpClientBinder. The use of this API +is now required. The modules cannot be used directly anymore. Here is an +example of how to use the binder: + + httpClientBinder(binder).bindHttpClient("foo", FooClient.class) + .withFilter(FirstFilter.class) + .withFilter(SecondFilter.class); + +Here is a simpler example to create a client for use with other platform +services. The client will pass the trace token when making requests: + + httpClientBinder(binder).bindHttpClient("foo", FooClient.class).withTracing(); + +Platform 0.63 + +* Fix bug that prevented sample and skeleton servers from starting + +Platform 0.62 + +* Removed all dependencies to Ning's Asynchronous HTTP client + +Since http-client-experimental now replaces the usage of Ning's Asynchronous +HTTP client, we've removed all existing references to that package. Rest +servers that upgrade to 0.62 may need to include it as an explicit dependency +if they don't want to upgrade. + +* Cleaned up dependency management + +There is now a library project that contains a single parent dependencyManagement +section for all poms to inherit from. This now includes REST Server Base. This +ensures that all platform libraries are built with a single set of versions that +produce fully compatible dependency closures with the services that consume them. + +* Multiple refactoring commits for readability + +There were multiple refactoring efforts to enhance readability. Some classes that +moved include: +RequestBuilder -> Request.Builder +EchoServlet is a separate class now +MediaType is now available as a class in Guava 12 + +* Addition of request filter support to HttpClient + +HttpClients may now bind in request filters, an example of such can be found by +following the code in com.proofpoint.http.client.TestHttpClientBinder. This +was added to support having the TraceToken added into outbound http client requests. + +* Removal of deprecated JavaUrlHttpClient + +* Removal of the experimental module, segmented into separate modules + +At this time, we will no longer have a single experimental module, please be aware +of classes that have the @Beta annotation going forward. Such classes are subject +to backwards incompatible changes. + +Platform 0.61 + +* Upgrade Platform and Rest Server Base pom to Guava 12.0 + +* Tests for keep alives with ApacheHttpClient + +Platform 0.60 + +* Fixed bug that prevents Rack servers from announcing + +Platform 0.59 + +* Hibernate validator replaced with direct dependency on Apache BVal + +We replaced the indirect use of the Hibernate validator with a direct +dependency on Apache BVal. Consequently, the dependency was removed +from the dependencyManagement section of the rest-server-base POM and +should be removed from the POM of any servers or libraries as it is +no longer required by Platform. + +* Fixed a bug preventing V1 Events from being able to be used with HttpRequestEvents + +* Check for duplicate classes in the dependency closure during builds + +Starting at 0.59, any rest-server-base dependent projects will have a build +step added that ensures there are no duplicates of classes or resources on +the classpath. Any duplicates will trigger a failure of the build and the +build output will include all the problematic classes or resources and which +packages they come from. Using the dependency:tree build goal can help with +finding these conflicts and resolving them. + +* Add JsonEventSerializer for external use + +JsonEventSerializer serializes a single event in v2 format using a JsonGenerator. +The existing JsonEventWriter is not easily consumed from external projects. + +* Add support for Rack servers to be able to announce with Discovery + +Starting with 0.59, the configuration property "rackserver.announcement" will +set the name for the rack server to announce to discovery. The remaining +configuration to announce with discovery is identical to rest-server-base +parented projects. + +Platform 0.58 + +* Fixed bug that prevents Rack applications from starting + +* As of release 0.55, the HttpServerModule has additional dependencies: + + - HttpEventModule + - DiscoveryModule + - JsonModule + + +Platform 0.57 + +* Fixed bug that prevents ServiceInventory from refreshing periodically + +Platform 0.56 + +* Fixed bug that causes HttpRequestEvents to not be published + +Platform 0.55 + +* In this release we have upgraded the following dependencies + + - Jackson 1.9.5 + +* HTTP request events + +We now record an event for each HTTP request processed. The HTTP request logging and +new event system now properly support the X-FORWARDED-FOR and X-FORWARDED-PROTO headers. + +* Redesigned HTTP client + +The existing HTTP client interface has been renamed to AsyncHttpClient and a new synchronous +HTTP client interface has been added in its place. Additionally, the underlying HTTP client +engine has been replaced with Apache HTTP Components. + +The HTTP client is still experimental. + + +Platform 0.54 + +This release removed the use of DocLava for generating javadocs for projects rest-server-base. +It was triggering a bug in javadoc that prevented some projects from building successfully. + + +Platform 0.53 + +* In this release we have upgraded the following dependencies + + - Jetty 8.1.1 + - Guava 11.0.2 + +Jetty 8.0.3 has a bug where connections will leak in CLOSE_WAIT state if clients hang up +prematurely. + +* Testing servlet filters + +When developing servlet filters for use with the platform HTTP server, it is now possible +to test these filters inside the TestingHttpServer. Bind the TestingHttpServerModule into your +test injector along with the module that binds your servlet filter. + +* Rack packaging (JRuby on Rails) + +The rack packaging has been substantially updated with many changes: + + - rewritten launcher that supports the new node config file + - support only Rails 3.1+ with asset pipeline + - add support for JMX over HTTP + - use Bundler to require dependencies when starting application + - no creation of .bundle directory when packaging gems + - various bug fixes including PATH_INFO, logging, locking, resources + +This release requires Rails 3.1+ rather than supporting arbitrary rack +applications. It also requires the build machine to have a Ruby installation +with all of the application gems installed. This is a temporary restriction +and will be fixed in future releases. + +The build process runs "bundle exec rake assets:precompile". Please verify +that this command succeeds before running the build. The following config +change in config/application.rb is likely required to prevent the command +from trying to connect to the configured production database: + + # Only partially load application when precompiling assets + config.assets.initialize_on_precompile = false + +The following config changes are also required for production mode in +config/environments/production.rb: + + # Serve static assets directly from Rails + config.serve_static_assets = true + + # Enable threaded mode + config.threadsafe! + +Future versions of the rack packaging may configure these automatically. + + +Platform 0.52 + +* Admin Http Server thread pool + +The thread pool for the admin http server (for jmx over http) listener has been +separated from the main http thread pool to prevent DoS if all http threads in the main +pool get stuck. + +The pool size can be controlled via the http-server.admin.threads.min and http-server.admin.threads.max +configuration properties. + +* Configurable http client timeouts + +Connection and read timeouts for http client are now configurable. For Discovery client, use +discovery.http-client.connect-timeout and discovery.http-client.read-timeout. For Event client, +use event.http-client.connect-timeout and event.http-client.read-timeout. + +* Add external address to node info + +Added node.external-address property to specify the external address (e.g., Internet routable) +to the JVM. HTTP and JMX services are announced using the existing name and a '-external' name, +so external clients can easily find the server. + +Platform 0.51 + +* Trace token support for http requests + +The http server can now deal with request trace tokens passed in via a +request header (X-Proofpoint-TraceToken). The trace token is recorded +in the request logs and is made available to application code via a +TraceTokenManager object. If no token is provided, a new one is created +automatically. + +To enable this functionality, simply add a dependency to +com.proofpoint.platform:trace-token and add TraceTokenModule to the list of +guice modules for your application. + +* Event fields support Map and Multimap + +In addition to Iterable, the event client now supports event fields of type +java.util.Map and com.google.common.collect.Multimap. The map key type must +be java.lang.String, while the value type can be any standard supported type +or nested type (i.e. any type that is supported by Iterable). + + +Platform 0.50 + +* Library Upgrades + +In this release we have upgraded the following dependencies + + - Guava 10.0.1 + - TestNG 6.2.1 + - Joda time 2.0 + - CGlib 2.2.2 + - Hibernate validation 4.2.0.Final + - log4j-over-slf4j 1.6.2 + - Logback 0.9.30 + +* Log configuration via JMX + +Log levels can now be configured via JMX under an mbean named +com.proofpoint.log:name=Logging. Enable this by adding LogJmxModule to your +Guice modules. The AllLevels attribute returns all explicitly configured +loggers (it excludes those with an inherited level). + + +Platform 0.49, Oct 14th 2011 + +* Library Upgrades + +In this release we have upgraded the following dependencies + + - Jetty 8.0.3 + - Jackson 1.9.1 + +* Http server critical bugs + +There are a number of bugs in versions of Jetty prior to 8.0.3 that can cause +the server to spin in a busy loop when using SSL or leak file descriptors under +certain conditions. + +* Admin port + +The http-based jmx connector now runs on an alternate admin port to avoid polluting +the request logs and stats when monitoring the server through jmx. For backwards +compatibility, the http listener is bound to a random port. The binding can be +overriden via the http-server.admin.port property and can be turned off via +http-server.admin.enabled. + +* Service inventory + +Discovery client now finds the location of available discovery servers by calling +out to a service inventory API. The location of the service inventory API is specified +via the service-inventory.uri configuration property and is provided automatically +when deploying with recent snapshot versions of Galaxy. + +This feature is experimental, so the old discovery.uri property is still supported. + +* PGP signing of artifacts + +Projects that inherit from rest-server-base are now automatically signed with PGP on +release. To get this working, the release machine and account needs to have a PGP key +and agent configured. + +See http://www.sonatype.com/people/2010/01/how-to-generate-pgp-signatures-with-maven/ +for more information. + +* Http server request stats + +The http server now exposes additional request stats via JMX under an mbean named +com.proofpoint.http:name=RequestStats. + +The available stats include: +- Request count + - Total + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average requests + per second +- Bytes Read/Written to connection + - Total (since server start) + - Exponentially-weighted per-request mean, min, max, percentiles (50, 90, 99, 99.9) + with a bias towards the past 5 minutes + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average bytes + per second +- Request time (ms) + - Total (since server start) + - Exponentially-weighted per-request mean, min, max, percentiles (50, 90, 99, 99.9) + with a bias towards the past 5 minutes + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average bytes + per second + +* Http client request stats + +Event and discovery clients now expose http request stats via JMX under the +com.proofpoint.discovery.client:name=DiscoveryClient and +com.proofpoint.event.client:name=EventClient mbeans. + +The available stats include: +- Request count + - Total + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average requests + per second +- Bytes Read/Written to connection + - Total (since server start) + - Exponentially-weighted per-request mean, min, max, percentiles (50, 90, 99, 99.9) + with a bias towards the past 5 minutes + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average bytes + per second +- Request scheduling time (ms) -- time for request to get picked up by an available worker thread. + - Total (since server start) + - Exponentially-weighted per-request mean, min, max, percentiles (50, 90, 99, 99.9) + with a bias towards the past 5 minutes + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average bytes + per second +- Scheduling time (ms) -- time for request to get picked up by an available worker thread. + - Total (since server start) + - Exponentially-weighted per-request mean, min, max, percentiles (50, 90, 99, 99.9) + with a bias towards the past 5 minutes + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average bytes + per second +- Request time (ms) -- time to send request data to remote server + - Total (since server start) + - Exponentially-weighted per-request mean, min, max, percentiles (50, 90, 99, 99.9) + with a bias towards the past 5 minutes + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average bytes + per second +- Response time (ms) -- time to read response + - Total (since server start) + - Exponentially-weighted per-request mean, min, max, percentiles (50, 90, 99, 99.9) + with a bias towards the past 5 minutes + - 1-minute, 5-minute and 15-minute exponentially-weighted moving average bytes + per second + + +Platform 0.48, Oct 5th 2011 + +* Library Upgrades + +In this release we have upgraded the following dependencies + + - Guava 10.0 + - Hector 0.8.0-2 + - Jersey 1.9.1 + - Jetty 8.0.1 + +Note: we have banned all prior versions of Guava since the former versioning +names are not understood by maven and can result in invalid combinations of +dependencies. + +* Jmx over HTTP + +One of the major problems we have with supporting our servers in a production +environment is accessing JMX through the firewall. JMX uses RMI by default +and RMI binds to two random ports, so we can run multiple servers on the same +instance. This configuration annoyance is compounded by the bidirectional +nature of the RMI protocol. When an RMI object is published, it encodes the +local address of the server on which the object is running. In EC2, each +server has a private IP for use within EC2 and a public address which we use for +connecting from out side of EC2. Since the RMI object can only contain one +address, we must choose between JMX being internally or externally accessible. + +We have eliminated both of these problems by writing a HTTP based JMX Remoteing +(JSR 160) connector. To add this to your server, simply add the JmxHttpRpcModule +to the Guice modules. The following command will connect jvisualvm to the server: + + jvisualvm --cp:a jmx-http-rpc-experimental.jar \ + --openjmx service:jmx:http://: + +Note: You must install the VisualVM-MBean plugin to see mbeans in jvisualvm + + + +Platform 0.47, Sept 29th 2011 + +* EventClient and DiscoveryClient + +After running some of our servers under load, we have found major memory leaks +in EventClient and DiscoveryClient due to the AsyncHttpClient. Unfortunately, +these leaks are difficult to fix due to the complexity of this code base, so we +have replaced the use of AsyncHttpClient in these libraries with the +experimental http client. + +* HttpClient + +We have written a vastly simplified event client interface that supports only the +features needed by EventClient and DiscoveryClient. The current +implementation uses java.net.URL internally but we expect this to change. This +code base is under active development and should not be used outside of the +platform until it is stabilizes. + +Platform 0.46, Sept 26th 2011 + +* Launcher script + +This release addresses one of the recent problems we've had with our servers +running in Galaxy. Galaxy splits the installation directory, which contains +our configuration files, from the data directory where the Java process runs. +This split makes it easy for Galaxy to upgrade servers without losing +persistent server data, but this split means it is impossible for the Java +program to locate configuration files in the installation etc directory. We +have fixed this problem by having the launcher script symlink the etc +directory from the installation directory into the data directory, so the Java +process can find the configuration using a relative path. + + +Platform 0.45, Sept 22th 2011 + +* oss.sonatype.org + +We are pleased to announce that starting with this release, all artifacts are +published to Maven Central. You can search for our artifacts here: + + http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.proofpoint.platform%22 + +* HttpServer + +We have added the request size to the http server log. The new log line format +follows: + + timeStamp remoteAddress method uri user agent status contentRead contentCount requestTime + +As a comparison the previous log line format was: + + timeStamp remoteAddress method uri user agent status contentCount requestTime + +Additionally, we have reenabled SSL renegotiation since the underlying JVM bug +has been fixed. + +* Launcher script + +Starting with this release, the preferred method for passing node specific +information (e.g., id, environment, and pool) to a server is the --node-config +command line option. The argument to this option must point to a file +containing key-value pairs encoded as 'key=value'. The properties in this file +are simply added to the Java command line as -D parameters when launching the +server. Parameters can still be passed to the server using -D parameters on +the launcher, and these will override any parameters in the node config file. + +* JRuby on Rails + +It is now possible to package a Rails application just like any other Java +platform application. The packaged application will use the platform http +server, logging, and over time we will extend the integration to support other +platform services. An instructional guide is currently under review and will +be included soon. + + +and run rack applications, such as rails, using the +platform http server. The platform logger is automatically provided to rack and rails, and other +platform facilities can be used from within JRuby. Simply set +rack-server-base-experimental as your parent pom, and you gain many of the same +features that exist in the rest-server-base parent pom. An instructional guide is +currently under review and will be included soon. + +* Event Client v2 + +The event client for the event v2 protocol has been mostly rewritten to fix +various bugs and annoyances. Additionally, we have have expanded the allowed +event fields to include java.util.Iterable and nested simple event types. For +Iterable, any supported type is allowed for the elements except Iterable. +Nested event types, can be any event type without a special field (e.g., +timestamp, uuid, or host) since these are only allowed in the envelope +of the protocol. For a full example see: + + com.proofpoint.event.client.NestedDummyEventClass + +* Embedded Cassandra + +Cassandra has been upgraded to 0.8.5 which addresses many internal bugs (see +https://svn.apache.org/repos/asf/cassandra/tags/cassandra-0.8.5/CHANGES.txt +for more information). Additional, work has been on startup reliability and +timeout configuration. + +One backwards incompatible change is the default partitioner is now +ByteOrderedPartitioner instead of RandomPartitioner since most of our +application are using range queries. + +* AsyncHttpClient + +We have upgraded AsyncHttpClient to version 1.6.5, but since this version still +contains a memory leak we have removed it from the event client. We are +several evaluating long term solutions to the memory leak and hope to have one +implemented in the next release. + +* DataSize helper methods + +The toBytes() and roundTo(Unit) helper method have been added to DataSize for +easily converting the size to a long. These methods will throw an exception if +the size is bigger than a long in the specified unit. + +* Removed Discovery JDBC + +The experimental discovery based JDBC driver has been removed. This was +originally introduced for supporting database discovery to rails, but this +feature has been rewritten in much simpler pure Ruby. + +* Bug fixes + - System out and err redirect to logging does not flush + - Fixed race condition in embedded Cassandra lifecycle + - Fixed error reporting in event client + - Fixed launcher issues with spawn using 'sh -c' on Linux + + +Platform 0.44, Sept 22th 2011 + +* Bad release + +Don't use this release, it is bad. + +Platform 0.43, Jun 10th 2011 + +* JRuby on Rails integration + +We've started the work support JRuby on Rails in the platform by adding a +servlet based implementation of Rack. Work will continue in future releases. + + +* Bug fixes + - Discovery client fails to reschedule refresh job in some cases + + + + +Platform 0.42, May 24th 2011 + +* Embedded Cassandra and testing utilities + +The code is part of the cassandra-experimental module +(com.proofpoint.platform:cassandra-experimental:0.42). The server is enabled by +adding CassandraModule to your Guice injector. Configuration for the server is +encapsulated in CassandraServerConfig. Code that needs to talk to the cassandra +server (e.g., Hector) should depend on CassandraServerInfo, which exposes a +method for obtaining the RPC port that Cassandra listens on. + +Here's an example of how to run a simple server with an embedded cassandra instance: + + + Bootstrap bootstrap = new Bootstrap(new NodeModule(), new CassandraModule()); + + try { + Injector injector = bootstrap.initialize(); + CassandraServerInfo info = injector.getInstance(CassandraServerInfo.class); + log.info("Cassandra server listening on port " + info.getRpcPort()); + } + catch (Throwable e) { + log.error(e); + // Cassandra creates non-daemon threads that will prevent the vm from shutting down + System.exit(1); + } + + + +We've also added some utilities to simplify unit testing of code that talks to +Cassandra. It is important to keep in mind that due to how Cassandra is +written, there can only be one embedded instance per VM, and it can only be +started once during the lifetime of the VM. + +To use Cassandra in unit tests, use CassandraServerSetup to initialize the +embedded instance. This should be done in @BeforeSuite and @AfterSuite methods +that call CassandraServerSetup.tryInitialize() and tryShutdown(), respectively. +It's important that this be done in every class that contains cassandra-based +tests. Otherwise, the initialization code will only run if you include the +class with these methods in your unit test execution. TryInitialize() and +tryShutdown() are designed to properly handle calls from multiple +@Before/AfterSuite methods. + +The server will be bound to a random port which can be obtained through the +CassandraServerInfo object. To get this object use either +CassandraServerSetup.getServerInfo() or TestingCassandraModule if your tests +require Guice. + +* DataSize + +We've introduced a DataSize class to the experimental module for dealing with +units of data in human-readable form (similar to Duration). It supports B, kB, +MB, GB, TB, PB and can handle conversions between these units. It's also fully +compatible with the configuration system. + +* ValidationAssertions + +We've also added some utilities to the experimental module to make easier to +test code that uses Bean Validation Framework annotations. +ValidationAssertions.assertValidates() and assertFailsValidation() can be used +to test that an object passes or fails validations as specified by its +annotations. Here's an example on how to use assertFailsValidation(): + + assertFailsValidation(config, "maxAge", "may not be null", NotNull.class) + +* Bug fixes + +The following issues have been fixed: + - Discovery client fails when consuming static announcements + - MySQL connection timeouts using the wrong unit (ms vs s) + - Jar manifest using the wrong file names when building snapshot versions diff --git a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/encoding/FileEncodingTest.java b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/encoding/FileEncodingTest.java index 60b08299422d..49b2211606b0 100644 --- a/platform/platform-tests/testSrc/com/intellij/openapi/vfs/encoding/FileEncodingTest.java +++ b/platform/platform-tests/testSrc/com/intellij/openapi/vfs/encoding/FileEncodingTest.java @@ -931,4 +931,13 @@ public class FileEncodingTest extends PlatformTestCase implements TestDialog { manager.setDefaultCharsetName(oldProject); } } + + public void testBigFileAutoDetectedAsTextMustDetermineItsEncodingFromTheWholeTextToMinimizePossibilityOfUmlautInTheEndMisdetectionError() { + VirtualFile vTestRoot = getTestRoot(); + VirtualFile file = vTestRoot.findChild("BIGCHANGES"); + assertNotNull(file); + + assertNull(file.getBOM()); + assertEquals(CharsetToolkit.UTF8_CHARSET, file.getCharset()); + } }