Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 2 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Scala Steward: Reformat with scalafmt 2.7.5
c549b943c5d68c27786a7aa1bbe70aa4f23e1710
2 changes: 1 addition & 1 deletion .scalafmt.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version = "2.0.1"
version = "2.7.5"
style = defaultWithAlign
maxColumn = 150
rewriteTokens = {
Expand Down
5 changes: 4 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,10 @@ lazy val dockerSettings = Seq(
dockerCommands := {
val cmds = dockerCommands.value
val injected = Seq(
Cmd("RUN", "apt update && apt install -y fuse libfuse2 libfuse-dev python3-pip && rm -rf /var/lib/apt/lists/*"), // TODO https://github.com/docker/for-mac/issues/3431
Cmd(
"RUN",
"apt update && apt install -y fuse libfuse2 libfuse-dev python3-pip && rm -rf /var/lib/apt/lists/*"
), // TODO https://github.com/docker/for-mac/issues/3431
Cmd("RUN", "echo 'user_allow_other' >> /etc/fuse.conf"),
Cmd(
"RUN",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import com.karasiq.shadowcloud.cache.ChunkCache
import com.karasiq.shadowcloud.model.Chunk

private[cache] class LArrayChunkCache(size: Int)(implicit ec: ExecutionContext) extends ChunkCache {
protected val cache = LArrayAsyncLRUCache[Chunk](size)
protected val cache = LArrayAsyncLRUCache[Chunk](size)
protected var lastChunk: Chunk = _

def readCached(chunk: Chunk, getChunk: () ⇒ Future[Chunk]): Future[Chunk] = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,20 @@ object SCConsoleConfig {
def load(): Config = {
val autoParallelismConfig = {
val cpuAvailable = Runtime.getRuntime.availableProcessors()
val parallelism = math.max(8, math.min(1, cpuAvailable / 2))
val parallelism = math.max(8, math.min(1, cpuAvailable / 2))
ConfigFactory.parseString(s"shadowcloud.parallelism.default = $parallelism")
}

val substitutionsConfig = ConfigFactory.parseResourcesAnySyntax("sc-substitutions")

val defaultConfig = ConfigFactory.defaultOverrides()
val defaultConfig = ConfigFactory
.defaultOverrides()
.withFallback(ConfigFactory.defaultApplication())
.withFallback(ConfigFactory.defaultReference())

val serverAppConfig = {
val fileConfig = {
val configFilePath = sys.props.getOrElse("shadowcloud.external-config", "shadowcloud.conf")
val configFilePath = sys.props.getOrElse("shadowcloud.external-config", "shadowcloud.conf")
val optionalConfFile = Paths.get(configFilePath)
if (Files.isRegularFile(optionalConfFile))
ConfigFactory.parseFile(optionalConfFile.toFile)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ class RegionDispatcherTest extends SCExtensionSpec with FlatSpecLike with Sequen

it should "create availability report" in {
val report = (testRegion ? GetFileAvailability(folder.files.head.copy(chunks = Seq(chunk)))).mapTo[GetFileAvailability.Success].futureValue
report.result.chunksByStorage shouldBe Map(testStorageId → Set(chunk))
report.result.chunksByStorage shouldBe Map(testStorageId → Set(chunk))
report.result.percentagesByStorage shouldBe Map(testStorageId → 100.0)
}

Expand Down Expand Up @@ -231,12 +231,15 @@ class RegionDispatcherTest extends SCExtensionSpec with FlatSpecLike with Sequen
resultDiff.time should be > oldDiff.time
}


override protected def beforeAll(): Unit = {
super.beforeAll()
storage ! StorageIndex.OpenIndex(testRegionId)
testRegion ! RegionDispatcher.AttachStorage(testStorageId, storageProps, storage, initialHealth)
awaitAssert(RegionDispatcher.GetHealth.unwrapFuture(testRegion ? RegionDispatcher.GetHealth).futureValue shouldBe 'fullyOnline, 30 seconds, 1 second)
awaitAssert(
RegionDispatcher.GetHealth.unwrapFuture(testRegion ? RegionDispatcher.GetHealth).futureValue shouldBe 'fullyOnline,
30 seconds,
1 second
)
}

private def storageUnsubscribe(): Unit = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import org.scalatest.{FlatSpecLike, SequentialNestedSuiteExecution}
import scala.concurrent.duration._

class RegionGCTest extends SCExtensionSpec with FlatSpecLike with SequentialNestedSuiteExecution {
val testRegionId = "regionGCTest"
val testRegionId = "regionGCTest"
val testStorageId = "regionGCTest"

"Region GC" should "delete broken chunks" in {
Expand Down Expand Up @@ -59,7 +59,6 @@ class RegionGCTest extends SCExtensionSpec with FlatSpecLike with SequentialNest
}
}


override protected def beforeAll(): Unit = {
super.beforeAll()
registerRegionAndStorages()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,16 @@ class RegionRepairTest extends SCExtensionSpec with FlatSpecLike with Sequential
sc.ops.region.writeChunk(testRegionId, chunk).futureValue shouldBe chunk
sc.ops.region.synchronize(testRegionId).futureValue

val (repairStream, repairResult) = TestSource.probe[RegionRepairStream.Request]
val (repairStream, repairResult) = TestSource
.probe[RegionRepairStream.Request]
.alsoTo(RegionRepairStream(sc.ops.region, sc.config.parallelism, sc.config.chunks.chunkSize))
.mapAsync(1)(_.result.future)
.toMat(TestSink.probe)(Keep.both)
.run()

repairStream.sendNext(RegionRepairStream.Request(testRegionId,
RegionRepairStream.Strategy.SetAffinity(ChunkWriteAffinity(Seq(testStorage1, testStorage2)))))
repairStream.sendNext(
RegionRepairStream.Request(testRegionId, RegionRepairStream.Strategy.SetAffinity(ChunkWriteAffinity(Seq(testStorage1, testStorage2))))
)

val repaired = repairResult.requestNext()
repaired shouldBe Seq(chunk)
Expand All @@ -51,7 +53,8 @@ class RegionRepairTest extends SCExtensionSpec with FlatSpecLike with Sequential

private[this] def registerRegionAndStorages(): Unit = {
sc.ops.supervisor.createRegion(testRegionId, sc.configs.regionConfig(testRegionId))
sc.ops.supervisor.createStorage(testStorage1, StorageProps.inMemory.copy(quota = Quota(ConfigProps.toConfig(ConfigProps("use-space-percent" → 33)))))
sc.ops.supervisor
.createStorage(testStorage1, StorageProps.inMemory.copy(quota = Quota(ConfigProps.toConfig(ConfigProps("use-space-percent" → 33)))))
sc.ops.supervisor.createStorage(testStorage2, StorageProps.inMemory)
sc.ops.supervisor.register(testRegionId, testStorage1)
sc.ops.supervisor.register(testRegionId, testStorage2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ import scala.concurrent.duration._
class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matchers {
import RegionSupervisor._

val testRegion = "testRegion"
val testRegion = "testRegion"
val testStorage = "testStorage"
val supervisor = system.actorOf(props, "supervisor")
val supervisor = system.actorOf(props, "supervisor")

"Region supervisor" should "add region" in {
supervisor ! CreateRegion(testRegion, CoreTestUtils.regionConfig("testRegion"))
Expand Down Expand Up @@ -78,7 +78,7 @@ class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matche

it should "resume storage" in {
supervisor ! ResumeStorage(testStorage)
val (regions, storages) = requestState()
val (regions, storages) = requestState()
val StorageStatus(`testStorage`, _, ActorState.Active(_), storageRegions) = storages(testStorage)
storageRegions shouldBe Set(testRegion)
regions.keySet shouldBe Set(testRegion)
Expand All @@ -95,7 +95,7 @@ class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matche

it should "resume region" in {
supervisor ! ResumeRegion(testRegion)
val (regions, storages) = requestState()
val (regions, storages) = requestState()
val RegionStatus(`testRegion`, _, ActorState.Active(_), regionStorages) = regions(testRegion)
regionStorages shouldBe Set(testStorage)
storages.keySet shouldBe Set(testStorage)
Expand All @@ -116,4 +116,3 @@ class RegionSupervisorTest extends SCExtensionSpec with FlatSpecLike with Matche
(regions, storages)
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,17 @@ class LZ4StreamsTest extends ActorSpec with ActorSpecImplicits with FlatSpecLike
val testBytes = TestUtils.indexedBytes._1

"LZ4" should "compress bytes" in {
val futureCompressed = Source.fromIterator(() ⇒ testBytes.grouped(100))
val futureCompressed = Source
.fromIterator(() ⇒ testBytes.grouped(100))
.via(LZ4Streams.compress)
.via(ByteStreams.concat)
.runWith(Sink.head)

val compressed = futureCompressed.futureValue
compressed should not be empty

val futureUncompressed = Source.fromIterator(() ⇒ compressed.grouped(33))
val futureUncompressed = Source
.fromIterator(() ⇒ compressed.grouped(33))
.via(LZ4Streams.decompress)
.via(ByteStreams.concat)
.runWith(Sink.head)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ class StreamCompressionTest extends ActorSpec with ActorSpecImplicits with FlatS
val testBytes = TestUtils.indexedBytes._1

def testCompression(uncompressed: ByteString): ByteString = {
val futureCompressed = Source.fromIterator(() ⇒ uncompressed.grouped(100))
val futureCompressed = Source
.fromIterator(() ⇒ uncompressed.grouped(100))
.via(StreamCompression.compress(compType))
.via(ByteStreams.concat)
.runWith(Sink.head)
Expand All @@ -30,7 +31,8 @@ class StreamCompressionTest extends ActorSpec with ActorSpecImplicits with FlatS
}

def testDecompression(compressed: ByteString, expected: ByteString): Unit = {
val futureUncompressed = Source.fromIterator(() ⇒ compressed.grouped(33))
val futureUncompressed = Source
.fromIterator(() ⇒ compressed.grouped(33))
.via(StreamCompression.decompress)
.via(ByteStreams.concat)
.runWith(Sink.head)
Expand Down Expand Up @@ -60,10 +62,10 @@ object StreamCompressionTest {
def readTestVector(compType: CompressionType.Value): (ByteString, ByteString) = {
ResourceUtils.getPathOption(s"compression-vectors/$compType") match {
case Some(path) ⇒
val inputStream = Files.newInputStream(path)
val inputStream = Files.newInputStream(path)
val objectInputStream = new ObjectInputStream(inputStream)
val uncompressed = objectInputStream.readObject().asInstanceOf[ByteString]
val compressed = objectInputStream.readObject().asInstanceOf[ByteString]
val uncompressed = objectInputStream.readObject().asInstanceOf[ByteString]
val compressed = objectInputStream.readObject().asInstanceOf[ByteString]
objectInputStream.close()
(uncompressed, compressed)

Expand All @@ -76,7 +78,7 @@ object StreamCompressionTest {
val fileName = testVectorsFolder.resolve(compType.toString)
if (!Files.exists(fileName)) {
if (!Files.isDirectory(testVectorsFolder)) Files.createDirectories(testVectorsFolder)
val outputStream = Files.newOutputStream(fileName)
val outputStream = Files.newOutputStream(fileName)
val objectOutputStream = new ObjectOutputStream(outputStream)
objectOutputStream.writeObject(uncompressed)
objectOutputStream.writeObject(compressed)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ class EncryptionModuleTest extends FlatSpec with Matchers {
}

private[this] def testEncryption(module: EncryptionModule): Unit = {
val data = TestUtils.randomBytes(100)
val data = TestUtils.randomBytes(100)
val parameters = module.createParameters()
val encrypted = module.encrypt(data, parameters)
val encrypted = module.encrypt(data, parameters)
// encrypted should not be data
encrypted.length should be >= data.length
val decrypted = module.decrypt(encrypted, parameters)
Expand All @@ -53,13 +53,13 @@ class EncryptionModuleTest extends FlatSpec with Matchers {

private[this] def runCrossTest(method1: EncryptionMethod, method2: EncryptionMethod): Unit = {
def toString(m: EncryptionMethod) = s"${m.provider.capitalize} (${m.algorithm})"
val module1 = modules.crypto.encryptionModule(method1)
val module2 = modules.crypto.encryptionModule(method2)
val module1 = modules.crypto.encryptionModule(method1)
val module2 = modules.crypto.encryptionModule(method2)
s"${toString(method1)}" should s"create compatible data for ${toString(method2)}" in {
val data = TestUtils.randomBytes(100)
val data = TestUtils.randomBytes(100)
val parameters = module1.createParameters()
val encrypted = module1.encrypt(data, parameters)
val decrypted = module2.decrypt(encrypted, parameters)
val encrypted = module1.encrypt(data, parameters)
val decrypted = module2.decrypt(encrypted, parameters)
decrypted shouldBe data
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
package com.karasiq.shadowcloud.test.crypto



import akka.util.ByteString
import com.karasiq.shadowcloud.model.crypto.HashingMethod
import com.karasiq.shadowcloud.test.utils.{ByteStringImplicits, CoreTestUtils}
import org.scalatest.{FlatSpec, Matchers}

class HashingModuleTest extends FlatSpec with Matchers with ByteStringImplicits {
val testData = ByteString("The testing facilities described up to this point were aiming at formulating assertions about a system’s behavior. If a test fails, it is usually your job to find the cause, fix it and verify the test again. This process is supported by debuggers as well as logging, where the Akka toolkit offers the following options:")
val testData = ByteString(
"The testing facilities described up to this point were aiming at formulating assertions about a system’s behavior. If a test fails, it is usually your job to find the cause, fix it and verify the test again. This process is supported by debuggers as well as logging, where the Akka toolkit offers the following options:"
)

val hashes = Map(
"SHA1" → "b608b0697e31e807fff8a37ae0d0292e62a8bbc2",
"SHA256" → "299a5a1fef2c67b6af326841684e020405978a34f7924025c1f5a188d808f08f",
"SHA512" → "46d69344d4fab2e20441d49a31ba94fbdba3d551fbbeb4646a00d9fd0e4d1a4b27c5c7ff9fcc992f0d88675e4de0dce1a8c8420298a60789d7a76ddf841d8088",
"SHA1" → "b608b0697e31e807fff8a37ae0d0292e62a8bbc2",
"SHA256" → "299a5a1fef2c67b6af326841684e020405978a34f7924025c1f5a188d808f08f",
"SHA512" → "46d69344d4fab2e20441d49a31ba94fbdba3d551fbbeb4646a00d9fd0e4d1a4b27c5c7ff9fcc992f0d88675e4de0dce1a8c8420298a60789d7a76ddf841d8088",
"Blake2b" → "f38fec1634ceaae44ca6a6ee6f493938b746a9d82581235ca6a748dc21ad865d"
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@ import com.karasiq.shadowcloud.test.utils.TestUtils

class IndexEncryptionTest extends FlatSpec with Matchers {
"Index encryption" should "create key hash" in {
IndexEncryption.getKeyHash(UUID.fromString("fb390d6a-545e-4ea7-a3ac-45e4a9223ebb"),
UUID.fromString("1ac2ad29-3edb-4f65-9142-cde158f06d28")) shouldBe (-1060735836)
IndexEncryption.getKeyHash(
UUID.fromString("fb390d6a-545e-4ea7-a3ac-45e4a9223ebb"),
UUID.fromString("1ac2ad29-3edb-4f65-9142-cde158f06d28")
) shouldBe (-1060735836)
}

it should "create nonce" in {
val nonce1 = TestUtils.indexedBytes._1.slice(0, 16)
val nonce2 = TestUtils.indexedBytes._1.slice(16, 32)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
package com.karasiq.shadowcloud.test.crypto



import com.karasiq.shadowcloud.crypto.SignModule
import com.karasiq.shadowcloud.model.crypto.{HashingMethod, SignMethod}
import com.karasiq.shadowcloud.test.utils.CoreTestUtils.modules
import com.karasiq.shadowcloud.test.utils.TestUtils
import org.scalatest.{FlatSpec, Matchers}

class SignModuleTest extends FlatSpec with Matchers {
val hashingMethod = HashingMethod("SHA-512")
val hashingMethod = HashingMethod("SHA-512")

runTest("RSA", 1024)

Expand All @@ -30,21 +28,21 @@ class SignModuleTest extends FlatSpec with Matchers {
}

private[this] def testSignature(module: SignModule): Unit = {
val data = TestUtils.randomBytes(100)
val data = TestUtils.randomBytes(100)
val parameters = module.createParameters()
val signature = module.sign(data, parameters)
val signature = module.sign(data, parameters)
module.verify(data, signature, parameters) shouldBe true
}

private[this] def runCrossTest(method1: SignMethod, method2: SignMethod): Unit = {
def toString(m: SignMethod) = s"${m.provider.capitalize} (${m.algorithm})"
val module1 = modules.crypto.signModule(method1)
val module2 = modules.crypto.signModule(method2)
val module1 = modules.crypto.signModule(method1)
val module2 = modules.crypto.signModule(method2)
s"${toString(method1)}" should s"create compatible signature for ${toString(method2)}" in {
val data = TestUtils.randomBytes(100)
val data = TestUtils.randomBytes(100)
val parameters = module1.createParameters()
val signature = module1.sign(data, parameters)
val result = module2.verify(data, signature, parameters)
val signature = module1.sign(data, parameters)
val result = module2.verify(data, signature, parameters)
result shouldBe true
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package com.karasiq.shadowcloud.test.index



import com.karasiq.shadowcloud.index.ChunkIndex
import com.karasiq.shadowcloud.test.utils.CoreTestUtils
import org.scalatest.{FlatSpec, Matchers}
Expand All @@ -15,19 +13,19 @@ class ChunkIndexTest extends FlatSpec with Matchers {
}

it should "delete chunk" in {
val index = ChunkIndex(Set(chunk))
val index = ChunkIndex(Set(chunk))
val index1 = index.deleteChunks(chunk)
index1.chunks shouldBe empty
}

it should "delete chunk without data" in {
val index = ChunkIndex(Set(chunk))
val index = ChunkIndex(Set(chunk))
val index1 = index.deleteChunks(chunk.withoutData)
index1.chunks shouldBe empty
}

it should "merge" in {
val index = ChunkIndex(Set(chunk))
val index = ChunkIndex(Set(chunk))
val chunk1 = CoreTestUtils.randomChunk
val index1 = ChunkIndex(Set(chunk1))
val merged = index.merge(index1)
Expand Down
Loading