From 4eebe5efca08b3cfc02d72776d3476eb561fa8ad Mon Sep 17 00:00:00 2001 From: Mikhail Mazurkevich Date: Tue, 3 Dec 2019 21:05:32 +0300 Subject: [PATCH] Revert "Extract appendDataWithoutCache method to the JPS specific collection" This reverts commit 312ebc40 GitOrigin-RevId: 0f8b960da79d9edc0ad36b0179cdd861721b0a44 --- .../IntIntPersistentMultiMaplet.java | 6 +- .../IntObjectPersistentMultiMaplet.java | 15 +++-- .../ObjectObjectPersistentMultiMaplet.java | 8 +-- .../storage/AbstractStateStorage.java | 25 ++++++-- .../storage/JpsPersistentHashMap.java | 62 ------------------- .../storage/OneToManyPathsMapping.java | 4 +- .../storage/OutputToTargetRegistry.java | 2 +- .../intellij/util/io/PersistentHashMap.java | 25 +++++++- 8 files changed, 66 insertions(+), 81 deletions(-) delete mode 100644 jps/jps-builders/src/org/jetbrains/jps/incremental/storage/JpsPersistentHashMap.java diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntIntPersistentMultiMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntIntPersistentMultiMaplet.java index 5bcb111ebd40..13e892151620 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntIntPersistentMultiMaplet.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntIntPersistentMultiMaplet.java @@ -6,11 +6,11 @@ import com.intellij.util.containers.SLRUCache; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.DataInputOutputUtil; import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.PersistentHashMap; import gnu.trove.TIntHashSet; import gnu.trove.TIntObjectProcedure; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.storage.BuildDataCorruptedException; -import org.jetbrains.jps.incremental.storage.JpsPersistentHashMap; import java.io.*; @@ -20,11 +20,11 @@ import java.io.*; public class IntIntPersistentMultiMaplet extends IntIntMultiMaplet { private static final TIntHashSet NULL_COLLECTION = new TIntHashSet(); private static final int CACHE_SIZE = 128; - private final JpsPersistentHashMap myMap; + private final PersistentHashMap myMap; private final SLRUCache myCache; public IntIntPersistentMultiMaplet(final File file, final KeyDescriptor keyExternalizer) throws IOException { - myMap = new JpsPersistentHashMap<>(file, keyExternalizer, new IntSetExternalizer()); + myMap = new PersistentHashMap<>(file, keyExternalizer, new IntSetExternalizer()); myCache = new SLRUCache(CACHE_SIZE, CACHE_SIZE) { @NotNull @Override diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMultiMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMultiMaplet.java index 42ed2eb12c46..e335e39a968c 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMultiMaplet.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/IntObjectPersistentMultiMaplet.java @@ -4,10 +4,10 @@ package org.jetbrains.jps.builders.java.dependencyView; import com.intellij.util.containers.SLRUCache; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.PersistentHashMap; import gnu.trove.TIntObjectProcedure; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.storage.BuildDataCorruptedException; -import org.jetbrains.jps.incremental.storage.JpsPersistentHashMap; import java.io.*; import java.util.Collection; @@ -19,7 +19,7 @@ import java.util.Collections; public class IntObjectPersistentMultiMaplet extends IntObjectMultiMaplet { private static final Collection NULL_COLLECTION = Collections.emptySet(); private static final int CACHE_SIZE = 128; - private final JpsPersistentHashMap> myMap; + private final PersistentHashMap> myMap; private final DataExternalizer myValueExternalizer; private final SLRUCache myCache; @@ -28,7 +28,7 @@ public class IntObjectPersistentMultiMaplet extends IntObjectMultiMaplet { final DataExternalizer valueExternalizer, final CollectionFactory collectionFactory) throws IOException { myValueExternalizer = valueExternalizer; - myMap = new JpsPersistentHashMap<>(file, keyExternalizer, + myMap = new PersistentHashMap<>(file, keyExternalizer, new CollectionDataExternalizer<>(valueExternalizer, collectionFactory)); myCache = new SLRUCache(CACHE_SIZE, CACHE_SIZE) { @NotNull @@ -82,7 +82,14 @@ public class IntObjectPersistentMultiMaplet extends IntObjectMultiMaplet { public void put(final int key, final Collection value) { try { myCache.remove(key); - myMap.appendDataWithoutCache(key, value); + myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { + @Override + public void append(DataOutput out) throws IOException { + for (V v : value) { + myValueExternalizer.save(out, v); + } + } + }); } catch (IOException e) { throw new BuildDataCorruptedException(e); diff --git a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java index 4b43c54171b6..f7c2208cfb8b 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java +++ b/jps/jps-builders/src/org/jetbrains/jps/builders/java/dependencyView/ObjectObjectPersistentMultiMaplet.java @@ -4,10 +4,10 @@ package org.jetbrains.jps.builders.java.dependencyView; import com.intellij.util.containers.SLRUCache; import com.intellij.util.io.DataExternalizer; import com.intellij.util.io.KeyDescriptor; +import com.intellij.util.io.PersistentHashMap; import gnu.trove.TObjectObjectProcedure; import org.jetbrains.annotations.NotNull; import org.jetbrains.jps.builders.storage.BuildDataCorruptedException; -import org.jetbrains.jps.incremental.storage.JpsPersistentHashMap; import java.io.*; import java.util.Collection; @@ -19,15 +19,15 @@ import java.util.Collections; public class ObjectObjectPersistentMultiMaplet extends ObjectObjectMultiMaplet{ private static final Collection NULL_COLLECTION = Collections.emptySet(); private static final int CACHE_SIZE = 128; - private final JpsPersistentHashMap> myMap; + private final PersistentHashMap> myMap; private final SLRUCache myCache; public ObjectObjectPersistentMultiMaplet(final File file, final KeyDescriptor keyExternalizer, final DataExternalizer valueExternalizer, final CollectionFactory collectionFactory) throws IOException { - myMap = new JpsPersistentHashMap<>(file, keyExternalizer, - new CollectionDataExternalizer<>(valueExternalizer, collectionFactory)); + myMap = new PersistentHashMap<>(file, keyExternalizer, + new CollectionDataExternalizer<>(valueExternalizer, collectionFactory)); myCache = new SLRUCache(CACHE_SIZE, CACHE_SIZE, keyExternalizer) { @NotNull @Override diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java index 9ddd680786dd..e6eaaf170da0 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/AbstractStateStorage.java @@ -7,13 +7,14 @@ import com.intellij.util.io.KeyDescriptor; import com.intellij.util.io.PersistentHashMap; import org.jetbrains.annotations.Nullable; +import java.io.DataOutput; import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.Iterator; public abstract class AbstractStateStorage implements StorageOwner { - private JpsPersistentHashMap myMap; + private PersistentHashMap myMap; private final File myBaseFile; private final KeyDescriptor myKeyDescriptor; private final DataExternalizer myStateExternalizer; @@ -83,7 +84,23 @@ public abstract class AbstractStateStorage implements StorageOwner { public void appendData(final Key key, final T data) throws IOException { synchronized (myDataLock) { - myMap.appendDataWithoutCache(key, data); + myMap.appendData(key, new PersistentHashMap.ValueDataAppender() { + @Override + public void append(DataOutput out) throws IOException { + myStateExternalizer.save(out, data); + } + }); + } + } + + public void appendDataWithoutCache(final Key key, final T data) throws IOException { + if (data != null) { + synchronized (myDataLock) { + myMap.appendDataWithoutCache(key, data); + } + } + else { + remove(key); } } @@ -113,9 +130,9 @@ public abstract class AbstractStateStorage implements StorageOwner { } - private JpsPersistentHashMap createMap(final File file) throws IOException { + private PersistentHashMap createMap(final File file) throws IOException { FileUtil.createIfDoesntExist(file); - return new JpsPersistentHashMap<>(file, myKeyDescriptor, myStateExternalizer); + return new PersistentHashMap<>(file, myKeyDescriptor, myStateExternalizer); } @Override diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/JpsPersistentHashMap.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/JpsPersistentHashMap.java deleted file mode 100644 index a5d2fef34638..000000000000 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/JpsPersistentHashMap.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -package org.jetbrains.jps.incremental.storage; - -import com.intellij.openapi.util.ThreadLocalCachedValue; -import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; -import com.intellij.util.io.DataExternalizer; -import com.intellij.util.io.DataOutputStream; -import com.intellij.util.io.KeyDescriptor; -import com.intellij.util.io.PersistentHashMap; -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.io.IOException; - -public class JpsPersistentHashMap extends PersistentHashMap { - public JpsPersistentHashMap(@NotNull File file, - @NotNull KeyDescriptor keyDescriptor, - @NotNull DataExternalizer valueExternalizer) - throws IOException { - super(file, keyDescriptor, valueExternalizer); - } - - /** - * This method is used to append value directly into the chunk without saving to cache. - * It can be used in case of non changed appended data like in JSP. The main goal for now - * is to avoid binary data changes in saving during JPS builds because of flushing cache. - */ - public final void appendDataWithoutCache(Key key, Value value) throws IOException { - synchronized (myEnumerator) { - try { - final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); - AppendStream appenderStream = ourFlyweightAppenderStream.getValue(); - appenderStream.setOut(bytes); - myValueExternalizer.save(appenderStream, value); - appenderStream.setOut(null); - appendDataWithoutCache(key, bytes); - } - catch (IOException ex) { - markCorrupted(); - throw ex; - } - } - } - - private static class AppendStream extends DataOutputStream { - private AppendStream() { - super(null); - } - - private void setOut(BufferExposingByteArrayOutputStream stream) { - out = stream; - } - } - - private static final ThreadLocalCachedValue ourFlyweightAppenderStream = new ThreadLocalCachedValue() { - @NotNull - @Override - protected AppendStream create() { - return new AppendStream(); - } - }; -} diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java index cf893327cc96..38d0eb55cc9e 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OneToManyPathsMapping.java @@ -37,12 +37,12 @@ public class OneToManyPathsMapping extends AbstractStateStorage boundPaths) throws IOException { - super.appendData(normalizePath(keyPath), normalizePaths(boundPaths)); + super.appendDataWithoutCache(normalizePath(keyPath), normalizePaths(boundPaths)); } @Nullable diff --git a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java index d00f2f35c492..abf4ed6e607b 100644 --- a/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java +++ b/jps/jps-builders/src/org/jetbrains/jps/incremental/storage/OutputToTargetRegistry.java @@ -64,7 +64,7 @@ public class OutputToTargetRegistry extends AbstractStateStorage extends PersistentEnumeratorDelegate< } } - protected void appendDataWithoutCache(Key key, @NotNull final BufferExposingByteArrayOutputStream bytes) { + /** + * This method is used to append value directly into the chunk without saving to cache. + * It can be used in case of non changed appended data like in JSP. The main goal for now + * is to avoid binary data changes in saving during JPS builds because of flushing cache. + */ + public final void appendDataWithoutCache(Key key, @NotNull Value value) throws IOException { + if (myIsReadOnly) throw new IncorrectOperationException(); + synchronized (myEnumerator) { + try { + final BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(); + AppendStream appenderStream = ourFlyweightAppenderStream.getValue(); + appenderStream.setOut(bytes); + myValueExternalizer.save(appenderStream, value); + appenderStream.setOut(null); + appendDataWithoutCache(key, bytes); + } + catch (IOException ex) { + myEnumerator.markCorrupted(); + throw ex; + } + } + } + + private void appendDataWithoutCache(Key key, @NotNull final BufferExposingByteArrayOutputStream bytes) { myEnumerator.lockStorage(); try { long previousRecord;