Testbed code to mimic the situation, and it does not exhibit the problem.
Note Had trouble pasting the code with some special symbols. In particular replaced < with & lt ; to keep the HTML happy (about line 16). Also increased the indent.
/** Test case to explore OOM issue when converting a large number of mutable.Set's to immutable.Set's
* The 'lazy val finalSet' logic converts each mutable.Set into an immutable.Set, releasing the mutable.Set in the process.
*
* The original expectation was that this would consume roughly the same amount of memory as before the conversion!!!
**/
import scala.collection.mutable
import scala.collection.immutable
case class TestSetOOM(numStrings:Int) {
import TestSetOOM._
// tmpSet is loaded with sizeOfSet unique strings when this class is initialized
var tmpSet:mutable.Set[String] = mutable.Set[String]()
// finalSet is an immutable.Set initialized when finalSet.size is invoked. tmpSet is converted, and then released
lazy val finalSet:immutable.Set[String] = { val tmp = tmpSet.toSet; tmpSet = null; tmp}
// Executes at initialization
for(i <- 0 until numStrings) tmpSet += getString
}
object TestSetOOM {
/////// The basic parameters controlling the tests ////////
val numInstances = 10000 // How many instances of TestSetOOM to create
val sizeOfString = 1000 // How large to make each test string (+ 16 for the number)
val sizeOfSet = 1000 // How many test strings to add to the 'tmpSet' within each instance
val gcEveryN = 1000 // During the conversion phase, request a GC & pause every N instances processed
// NOTE: Would REALLY love to have a method to FORCE a complete, deep GC for use
// in exactly test routines such as this!!!
val pauseMillis = 100 // Number of milliseconds to pause during the gdEveryN, can be zero for no pause
// The hope is that if we pause the main thread then the JVM may actually run the GC
val reportAsMB = true // True == show memory reports as MB, false as GBs
//////// ... end of basic parameters ... ////////
val baseData = numInstances.toLong * sizeOfSet * (sizeOfString + 16)
def main(ignored:Array[String]):Unit = {
var instances:List[TestSetOOM] = Nil // Create numInstances and prepend to this list
for(i 0) Thread.sleep(pauseMillis)
ln("")
ln(f"Instances: $numInstances%,d, Size Of mutable.Set: $sizeOfSet%,d, Size of String: ${sizeOfString + 16}%,d == ${MBorGB(baseData)} base data size")
ln("")
ln(s" BASELINE -- $memUsedStr -- after initialization of all test data")
var dummy = 0L
var cnt = 0
instances.foreach { item =>
dummy += item.finalSet.size // Forces the conversion
cnt += 1
if(gcEveryN > 0 && (cnt % gcEveryN) == 0){
runtime.gc
if(pauseMillis > 0) Thread.sleep(pauseMillis)
ln(f"$cnt%,11d converted -- $memUsedStr ")
}
}
ln("")
ln(s" FINAL -- $memUsedStr")
}
val runtime = Runtime.getRuntime
// Get a memory report in either MBs or GBs
def memUsedStr = { val max = runtime.maxMemory
val ttl = runtime.totalMemory
val free = runtime.freeMemory
val used = ttl - free
s"Memory -- Max: ${MBorGB(max)}, Total: ${MBorGB(ttl)}, Used: ${MBorGB(used)}"
}
def MBorGB(amount:Long) = { val amt = amount / (if(reportAsMB) 1000000 else 1000000000L)
val as = if(reportAsMB) "MB" else "GB"
f"$amt%,9d $as"
}
// Generate strings with leading unique numbers as test data so the compiler cannot
// recognize as equal and memoize just one if they were all the same!
val emptyString = "X" * sizeOfString
var numString = 0L
def getString = { numString += 1
f"${numString}%,16d$emptyString"
}
def ln(str:String) = println(str)
}
Output of execution
Instances: 10,000, Size Of mutable.Set: 1,000, Size of String: 1,016 == 10,160 MB base data size
BASELINE -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 10,746 MB -- after initialization of all test data
1,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 10,783 MB
2,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 10,825 MB
3,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 10,867 MB
4,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 10,909 MB
5,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 10,951 MB
6,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 10,992 MB
7,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 11,034 MB
8,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 11,076 MB
9,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 11,117 MB
10,000 converted -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 11,158 MB
FINAL -- Memory -- Max: 15,032 MB, Total: 15,032 MB, Used: 11,162 MB
