goldcollection
/
muwire
mirror of https://github.com/zlatinb/muwire
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch to LAX Json parser and parallelize the stream after filtering

pull/62/head
Zlatin Balevsky 2021-06-09 12:50:00 +01:00
parent aa2b80cc0d
commit 1e7e30dee2
No known key found for this signature in database
GPG Key ID: A72832072D525E41
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
core/src/main/groovy/com/muwire/core/files/PersisterFolderService.groovy
View File

@ -5,6 +5,7 @@ import com.muwire.core.filefeeds.UIFilePublishedEvent
import com.muwire.core.filefeeds.UIFileUnpublishedEvent import com.muwire.core.filefeeds.UIFileUnpublishedEvent
import com.muwire.core.util.DataUtil import com.muwire.core.util.DataUtil
import groovy.json.JsonOutput import groovy.json.JsonOutput
import groovy.json.JsonParserType
import groovy.json.JsonSlurper import groovy.json.JsonSlurper
import groovy.util.logging.Log import groovy.util.logging.Log
@ -137,16 +138,16 @@ class PersisterFolderService extends BasePersisterService {
int loaded = 0 int loaded = 0
AtomicInteger failed = new AtomicInteger() AtomicInteger failed = new AtomicInteger()
Stream<Path> stream = Files.walk(location.toPath()) Stream<Path> stream = Files.walk(location.toPath())
if (core.muOptions.plugin) stream = stream.filter({
stream = stream.parallel()
stream.filter({
it.getFileName().toString().endsWith(".json") it.getFileName().toString().endsWith(".json")
}) })
.forEach({ if (core.muOptions.plugin)
stream = stream.parallel()
stream.forEach({
log.fine("processing path $it") log.fine("processing path $it")
def slurper = new JsonSlurper() def slurper = new JsonSlurper(type: JsonParserType.LAX)
try { try {
def parsed = slurper.parse it.toFile() def parsed = slurper.parse(it.toFile())
def event = fromJsonLite parsed def event = fromJsonLite parsed
if (event == null) return if (event == null) return