Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Firestore: Optimize local cache sync when resuming a query that had docs deleted #4982

Merged
merged 49 commits into from
May 8, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
b58e756
Update protos to include bloom filter (#4564)
milaGGL Jan 17, 2023
1b0b3ed
Merge branch 'master' into mila/BloomFilter
milaGGL Jan 17, 2023
fd907e5
Merge branch 'master' into mila/BloomFilter
milaGGL Jan 18, 2023
b306135
Implement BloomFilter class (#4524)
milaGGL Jan 20, 2023
82280ca
Merge branch 'master' into mila/BloomFilter
milaGGL Jan 26, 2023
2139f5e
Add expected count to target (#4574)
milaGGL Jan 27, 2023
078c414
Apply bloom filter on existence filter mismatch (#4601)
milaGGL Feb 9, 2023
3862731
Merge branch 'master' into mila/BloomFilter
milaGGL Feb 9, 2023
9e7023d
Merge branch 'master' into mila/BloomFilter
milaGGL Feb 17, 2023
7ea3c34
Update TargetData.java
milaGGL Feb 27, 2023
8696c17
Add integration test for bloom filter (#4696)
milaGGL Feb 28, 2023
a9a2b42
Merge branch 'master' into mila/BloomFilter
milaGGL Mar 9, 2023
8db1af7
Merge branch 'master' into mila/BloomFilter
milaGGL Mar 9, 2023
9f66dea
fromat
milaGGL Mar 9, 2023
70326ee
Update MockDatastore.java
milaGGL Mar 10, 2023
3d406f2
Merge remote-tracking branch 'origin/master' into HEAD
dconeybe Mar 11, 2023
9e889fe
Merge branch 'master' into mila/BloomFilter
milaGGL Mar 13, 2023
97dafee
Update WatchChangeAggregator.java
milaGGL Mar 14, 2023
90d57dd
Merge branch 'master' into mila/BloomFilter
milaGGL Mar 14, 2023
1bbf61b
update queryTest to be consistent with master
milaGGL Mar 14, 2023
06479a0
Add new goog-listen-tag for bloom filter (#4777)
milaGGL Mar 16, 2023
59f2859
Update the integration test to verify that bloom filter averted full …
dconeybe Mar 16, 2023
443289c
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe Mar 18, 2023
9716326
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe Mar 20, 2023
36a1643
Merge commit '687e079d401db1a75d2559038b879726a476ca3e' into mila/Blo…
dconeybe Mar 24, 2023
ecda2ba
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe Mar 24, 2023
79cc918
Merge branch 'master' into mila/BloomFilter
milaGGL Mar 27, 2023
d63be2b
Improve bloom filter application test coverage (#4828)
milaGGL Mar 30, 2023
8c6677f
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe Apr 7, 2023
158c084
QueryTest.java: Remove check for `getTargetBackend() != NIGHTLY` sinc…
dconeybe Apr 11, 2023
c76b67c
Port spec test changes from https://github.com/firebase/firebase-js-s…
dconeybe Apr 24, 2023
0f870e9
Port spec test changes from https://github.com/firebase/firebase-js-s…
dconeybe Apr 24, 2023
7202f9d
Merge branch 'master' into dconeybe/SpecTestPortJsSdkPr7021
dconeybe Apr 24, 2023
c8562d4
Merge remote-tracking branch 'remotes/origin/dconeybe/SpecTestPortJsS…
dconeybe Apr 24, 2023
e1ac631
regenerate spec tests json from js sdk
dconeybe Apr 25, 2023
766f99a
SpecTestCase.java: update to handle `targetPurpose` as a string, not …
dconeybe Apr 25, 2023
48a082b
Merge remote-tracking branch 'remotes/origin/dconeybe/SpecTestPortJsS…
dconeybe Apr 25, 2023
fa399de
Merge remote-tracking branch 'origin/master' into SpecTestPortJsSdkPr…
dconeybe Apr 25, 2023
0e9601d
Merge remote-tracking branch 'remotes/origin/dconeybe/SpecTestPortJsS…
dconeybe Apr 25, 2023
793bfc4
Merge remote-tracking branch 'remotes/origin/dconeybe/SpecTestPortJsS…
dconeybe Apr 25, 2023
105000a
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe Apr 25, 2023
98f62ad
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe Apr 27, 2023
6b22865
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe May 2, 2023
0a67ba1
Merge remote-tracking branch 'origin/master' into mila/BloomFilter
dconeybe May 5, 2023
9c12d19
Merge branch 'master' into mila/BloomFilter
milaGGL May 5, 2023
2573baa
Update CHANGELOG.md
milaGGL May 5, 2023
cc0428a
update CHANGELOG + format
milaGGL May 8, 2023
d0849f4
Random improvements. (#4986)
dconeybe May 8, 2023
d9a59e2
Update CHANGELOG.md
milaGGL May 8, 2023
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Random improvements. (#4986)
  • Loading branch information
dconeybe authored May 8, 2023
commit d0849f42833d0602e524c4b176861a986d377393
Original file line number Diff line number Diff line change
Expand Up @@ -17,42 +17,82 @@
import android.util.Base64;
import androidx.annotation.NonNull;
import androidx.annotation.VisibleForTesting;
import com.google.protobuf.ByteString;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

public class BloomFilter {
public final class BloomFilter {
private final int bitCount;
private final byte[] bitmap;
private final ByteString bitmap;
private final int hashCount;
private final MessageDigest md5HashMessageDigest;

public BloomFilter(@NonNull byte[] bitmap, int padding, int hashCount) {
if (bitmap == null) {
throw new NullPointerException("Bitmap cannot be null.");
}
/**
* Creates a new {@link BloomFilter} with the given parameters.
*
* @param bitmap the bitmap of the bloom filter; must not be null.
* @param padding the padding, in bits, of the last byte of the bloom filter; must be between 0
* (zero) and 7, inclusive; must be 0 (zero) if {@code bitmap.length==0}.
* @param hashCount The number of hash functions to use; must be strictly greater than zero; may
* be 0 (zero) if and only if {@code bitmap.length==0}.
*/
public BloomFilter(@NonNull ByteString bitmap, int padding, int hashCount) {
if (padding < 0 || padding >= 8) {
throw new BloomFilterException("Invalid padding: " + padding);
throw new IllegalArgumentException("Invalid padding: " + padding);
}
if (hashCount < 0) {
throw new BloomFilterException("Invalid hash count: " + hashCount);
throw new IllegalArgumentException("Invalid hash count: " + hashCount);
}
if (bitmap.length > 0 && hashCount == 0) {
if (bitmap.size() > 0 && hashCount == 0) {
// Only empty bloom filter can have 0 hash count.
throw new BloomFilterException("Invalid hash count: " + hashCount);
throw new IllegalArgumentException("Invalid hash count: " + hashCount);
}
if (bitmap.length == 0 && padding != 0) {
if (bitmap.size() == 0 && padding != 0) {
// Empty bloom filter should have 0 padding.
throw new BloomFilterException(
throw new IllegalArgumentException(
"Expected padding of 0 when bitmap length is 0, but got " + padding);
}

this.bitmap = bitmap;
this.hashCount = hashCount;
this.bitCount = bitmap.length * 8 - padding;
this.bitCount = bitmap.size() * 8 - padding;
this.md5HashMessageDigest = createMd5HashMessageDigest();
}

/**
* Creates an instance of {@link BloomFilter} with the given arguments, throwing a well-defined
* exception if the given arguments do not satisfy the requirements documented in the {@link
* BloomFilter} constructor.
*/
public static BloomFilter create(@NonNull ByteString bitmap, int padding, int hashCount)
throws BloomFilterCreateException {
if (padding < 0 || padding >= 8) {
throw new BloomFilterCreateException("Invalid padding: " + padding);
}
if (hashCount < 0) {
throw new BloomFilterCreateException("Invalid hash count: " + hashCount);
}
if (bitmap.size() > 0 && hashCount == 0) {
// Only empty bloom filter can have 0 hash count.
throw new BloomFilterCreateException("Invalid hash count: " + hashCount);
}
if (bitmap.size() == 0 && padding != 0) {
// Empty bloom filter should have 0 padding.
throw new BloomFilterCreateException(
"Expected padding of 0 when bitmap length is 0, but got " + padding);
}

return new BloomFilter(bitmap, padding, hashCount);
}

/** Exception thrown by {@link #create} if the given arguments are not valid. */
public static final class BloomFilterCreateException extends Exception {
public BloomFilterCreateException(String message) {
super(message);
}
}

@VisibleForTesting
int getBitCount() {
return this.bitCount;
Expand Down Expand Up @@ -146,7 +186,7 @@ private static long unsignedRemainder(long dividend, long divisor) {
/** Return whether the bit at the given index in the bitmap is set to 1. */
private boolean isBitSet(int index) {
// To retrieve bit n, calculate: (bitmap[n / 8] & (0x01 << (n % 8))).
byte byteAtIndex = this.bitmap[index / 8];
byte byteAtIndex = this.bitmap.byteAt(index / 8);
int offset = index % 8;
return (byteAtIndex & (0x01 << offset)) != 0;
}
Expand All @@ -159,7 +199,7 @@ public String toString() {
+ ", size="
+ bitCount
+ ", bitmap=\""
+ Base64.encodeToString(bitmap, Base64.NO_WRAP)
+ Base64.encodeToString(bitmap.toByteArray(), Base64.NO_WRAP)
+ "\"}";
}
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.google.firebase.firestore.remote.WatchChange.ExistenceFilterWatchChange;
import com.google.firebase.firestore.remote.WatchChange.WatchTargetChange;
import com.google.firebase.firestore.util.Logger;
import com.google.protobuf.ByteString;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand Down Expand Up @@ -252,14 +253,14 @@ private BloomFilterApplicationStatus applyBloomFilter(
return BloomFilterApplicationStatus.SKIPPED;
}

byte[] bitmap = unchangedNames.getBits().getBitmap().toByteArray();
ByteString bitmap = unchangedNames.getBits().getBitmap();
BloomFilter bloomFilter;

try {
bloomFilter =
new BloomFilter(
BloomFilter.create(
bitmap, unchangedNames.getBits().getPadding(), unchangedNames.getHashCount());
} catch (BloomFilterException e) {
} catch (BloomFilter.BloomFilterCreateException e) {
Logger.warn(
LOG_TAG,
"Applying bloom filter failed: ("
Expand Down
Loading