Commit ae8d3b63 by Tomasz Ciesielczyk

logging improvements

parent 3a5d805c
......@@ -5,11 +5,13 @@ import psnc.m2dc.rtm.model.rest.ResourceAction
import psnc.m2dc.rtm.model.rpm.ManagementMethodType
import psnc.m2dc.rtm.pcm.algorithm.Utils
import psnc.m2dc.rtm.rpm.RPMApi
import psnc.m2dc.rtm.utils.logInfo
import psnc.m2dc.rtm.utils.toJson
/**
* Author: Tomek
* Reent power actions applied to resources
*/
object ActionCache {
......@@ -17,38 +19,26 @@ object ActionCache {
//#region properties
private val lock = Any()
// /**
// * performance cache
// */
// private val currentPerformance: HashMap<String, Double> =
//
// RTMProperties.storage.get<HashMap<String, Double>>(
// StorageKeys.PERFORMANCE_SETTINGS, HashMap())
//#endregion
override fun toString(): String {
return StorageKeys.PERFORMANCE_SETTINGS.getMap().toJson()
}
//#region performance cache
fun setPerformance(resourceId: String, performance: Double, profile: String? /* = null*/) {
synchronized(this.lock) {
// printConsole(LightStorage.instance.listIndex("PCM_PERFORMANCE_SETTINGS")!!.toJson())
if(resourceId.length<3){
(Exception().printStackTrace())
throw NullPointerException("short id")
}
// if(resourceId.length<3){
// (Exception().printStackTrace())
// throw NullPointerException("short id")
// }
StorageKeys.PERFORMANCE_SETTINGS[resourceId] = performance
if (profile != null) {
StorageKeys.PROFILE_MAP[resourceId] = profile
} else {
StorageKeys.PROFILE_MAP.delete(resourceId)
}
// printConsole(LightStorage.instance.listIndex("PCM_PERFORMANCE_SETTINGS")!!.toJson())
// this.currentPerformance[resourceId] = performance
// RTMProperties.storage.put(StorageKeys.PERFORMANCE_SETTINGS, this.currentPerformance)
}
}
......@@ -76,12 +66,12 @@ object ActionCache {
/**
* reset power settings on selected @nodes
* or reset power settings on previously defined settings( based on stored actions) if @nodes is null
* restore power settings on selected @nodes
* @param nodes - node id list
*/
fun resetPowerSettings(nodes: Collection<String>? = null) {
fun restorePowerSettings(nodes: Collection<String>? = null) {
// synchronized(this.lock) {
logInfo("Restore power settings ")
(nodes?.map { it to 100.0 }?.toMap() ?: StorageKeys.PERFORMANCE_SETTINGS.getMap()).forEach({ it ->
val resourceId = it.key
val performance = it.value
......@@ -105,6 +95,35 @@ object ActionCache {
})
// }
}
/**
* reset settings on selected node - resend previously defined setting
* @param nodes - node id list
*/
fun resetPowerSettings( ) {
// synchronized(this.lock) {
logInfo("Reset power settings ")
StorageKeys.PERFORMANCE_SETTINGS.getMap().forEach({ it ->
val resourceId = it.key
val performance = it.value
when (performance) {
PowerSettings.SUSPEND, PowerSettings.POWER_OFF -> {
val action = ResourceAction(resourceId, ManagementMethodType.turn_on)
RPMApi.instance().startAction(action)
}
else -> {
//set
Utils.getPropertyAction(resourceId, performance, this.getProfile(resourceId))?.let {
it.forEach { action ->
RPMApi.instance().startAction(action)
}
}
}
}
})
// }
}
//#endregion
......
......@@ -13,8 +13,7 @@ import kotlin.Comparator
import kotlin.collections.HashMap
/**
* param: aheadBuffer -> length of the predicted load in minutes which is taken into consideration in the node
* priority
* current state of appliance
*/
class CurrentState(val powerUsage: Double) {
data class NodeState(val node: Resource, val powerUsage: Double, private val currentPerformance: Double, val currentProfile: String?, private var newPerformance: Double = currentPerformance) {
......@@ -33,12 +32,11 @@ class CurrentState(val powerUsage: Double) {
private val currentTimeInterval = TimeInterval()
var currentPowerOverLimit = PCMApi.Instance.getPowerLimit()?.let { this.powerUsage - it } ?: 0.0
var eval = Evaluation(currentPowerOverLimit,
StorageKeys.POWER_SETTING_EVALUATION_MODE)
var eval = Evaluation(currentPowerOverLimit, StorageKeys.POWER_SETTING_EVALUATION_MODE)
val unusedMachines: ArrayList<NodeUtil> = ArrayList()
private var activeMachines: ArrayList<NodeUtil> = ArrayList()
fun getNodeUtil(resourceId: String):NodeUtil? {
fun getNodeUtil(resourceId: String): NodeUtil? {
return activeMachines.find { it.nodeId == resourceId }
?: this.unusedMachines.find { it.nodeId == resourceId }
}
......@@ -60,29 +58,23 @@ class CurrentState(val powerUsage: Double) {
val powerMap = RPMApi.instance().getPowerUsageList()
DAO.Resource.getNodes().values.forEach({ node ->
val nodeUtil = currentUtil[node.id]
val power = nodeUtil?.powerUsage ?: powerMap[node.id] ?: {
var power = nodeUtil?.powerUsage ?: powerMap[node.id] ?: {
logError("Power Usage not provided for ${node.id}");0.0
}.invoke()
// printConsole("xxx: power for${node.id}= $power")
val currentPerf = ActionCache.getPerformanceSetting(node.id)
val currentProfile = nodeUtil?.loadProfile ?: ActionCache.getProfile(node.id)
nodeMap[node.id] = NodeState(node, power, currentPerf, currentProfile)
val model = DAO.Resource.getComponentPowerModel(node.typeId, null)!!
var currentUtilization = nodeUtil?.utilization ?: model.getUtil(power, currentPerf)
// val util = if (PCMApi.instance().getPowerCappingStrategy() == PowerCappingStrategy.EAGER)
// 100.0
// else
// nodeUtil?.utilization ?: model.getUtil(power, currentPerf)
//// printConsole("util for: ${node.id}- $util/$power")
if (PCMApi.Instance.getPowerCappingStrategy() == PowerCappingStrategy.EAGER) {
currentUtilization = /* assume maximum utilization*/100.0
power =/*scale power:*/power * 100 / currentUtilization
}
var newNodeUtil = NodeUtil(node.id, currentUtilization, power, currentProfile, null, currentTimeInterval)
// currentUtil[node.id] = newNodeUtil
if (newNodeUtil.isInUse(currentTimeInterval)) {
if (NodeUtil.isActive(newNodeUtil.utilization)) {
this.activeMachines.add(newNodeUtil)
val priority = StorageKeys.NODE_PRIORITIES_MAP[node.id]?.weight ?: 0.5
if (PCMApi.Instance.getPowerCappingStrategy() == PowerCappingStrategy.EAGER) {
newNodeUtil = NodeUtil(node.id, 100.0, power * 100 / currentUtilization, currentProfile, null, currentTimeInterval)
}
//TODO: przy eager mode przeliczyc overlimit i zuzucia renergi
val state = StateDefinition(model, newNodeUtil.utilization!!, currentPerf, priority)
if (this.canSetPerformance(node.id))
eval.addState(node.id, state)
......@@ -94,7 +86,6 @@ class CurrentState(val powerUsage: Double) {
this.orderUnusedMachines()
this.orderMachines()
// printConsole("${this.activeMachines} unused: ${this.unusedMachines}")
} catch (ex: Exception) {
ex.printStackTrace()
......
......@@ -7,10 +7,7 @@ import psnc.m2dc.rtm.model.rpm.Resource
import psnc.m2dc.rtm.properties.RTMProperties
import psnc.m2dc.rtm.properties.keys.Keys
import psnc.m2dc.rtm.rpm.RPMApi
import psnc.m2dc.rtm.utils.logError
import psnc.m2dc.rtm.utils.logWarn
import psnc.m2dc.rtm.utils.printConsole
import psnc.m2dc.rtm.utils.toJson
import psnc.m2dc.rtm.utils.*
import java.nio.file.Paths
import java.util.*
......@@ -22,14 +19,25 @@ abstract class RTMAlgorithm(currentState: CurrentState) {
fun saveState() {
if (Keys.RTM.dump_current_state.getBoolean()) {
nextDumpIndex?.let {idx->
nextDumpIndex?.let { idx ->
this.state.toJson(RTMProperties.parsePath("${RTMProperties.basePath}/dmp/${this::class.java.simpleName}_$idx.dmp").toString())
}?: logWarn("invalid dump dump_current_state_count: ${ Keys.RTM.dump_current_state_count}")
} ?: logWarn("invalid dump dump_current_state_count: ${Keys.RTM.dump_current_state_count}")
}
}
fun dumpSettings(settings: PowerSettings) {
nextSettingIndex?.let { idx ->
val s = settings.getResources().joinToString(separator = "\n", prefix = "[", postfix = "]") { r ->
val actions = settings.getActions(r)
"${r.id} = ${actions.performance}: ${actions.toJson()}"
}
s.store(RTMProperties.parsePath("${RTMProperties.basePath}/dmp/${this::class.java.simpleName}_$idx.dmp").toString())
} ?: logWarn("dumpSettings: invalid dump dump_current_state_count: ${Keys.RTM.dump_current_state_count}")
}
var state: CurrentState = currentState
private set
......@@ -37,6 +45,23 @@ abstract class RTMAlgorithm(currentState: CurrentState) {
companion object {
private val lock = Any()
private var currentDumpIndex = -1
private var currentSettingsIndex = -1
val nextSettingIndex: Long?
get() = synchronized(lock) {
if (Keys.RTM.dump_current_state_count.get() == -1) {
Date().time
} else {
val count = Keys.RTM.dump_current_state_count.get()
if (count == 0)
null
else {
currentSettingsIndex = (currentSettingsIndex + 1) % count
currentSettingsIndex.toLong()
}
}
}
val nextDumpIndex: Long?
get() = synchronized(lock) {
if (Keys.RTM.dump_current_state_count.get() == -1) {
......
......@@ -6,6 +6,7 @@ import psnc.m2dc.rtm.RTMAlgorithm
import psnc.m2dc.rtm.RTMApi
import psnc.m2dc.rtm.model.rpm.Resource
import psnc.m2dc.rtm.pcm.algorithm.Solver
import psnc.m2dc.rtm.utils.logInfo
import psnc.m2dc.rtm.utils.logWarn
import psnc.m2dc.rtm.utils.printConsole
import psnc.m2dc.rtm.utils.toJson
......@@ -43,10 +44,10 @@ class ESMAlgorithm(state: CurrentState,val algorithm: Solver.Algorithm) : RTMAlg
result.putAll(unusedNodesSuspendActions())
val suspendSavings = unusedResourcesSuspendSavings()
printConsole("esm powerbudget: ${-this.state.eval.powerOverLimit}")
logInfo("esm powerbudget: ${-this.state.eval.powerOverLimit}")
printConsole(RTMApi.getNodePerformance().toJson())
setSolver(suspendSavings)
printConsole("esm powerbudget: ${-this.state.eval.powerOverLimit}")
// printConsole("esm powerbudget: ${-this.state.eval.powerOverLimit}")
solver.solve()?.let {
it.forEach {
val state = this.state.nodeMap[it.key]!!.setPerformance(it.value)
......
......@@ -57,6 +57,7 @@ class ESMWorker private constructor(val state: CurrentState) : RTMWorker() {
notify(actions)
if(Keys.RTM.debug_actions.get()){
printConsole(actions.toJson())
this.algorithm.dumpSettings(actions)
}
else {
startActions(actions,this.state)
......
......@@ -7,7 +7,6 @@ import psnc.m2dc.rtm.model.rpm.Resource
import psnc.m2dc.rtm.pcm.algorithm.Solver
import psnc.m2dc.rtm.utils.logInfo
import psnc.m2dc.rtm.utils.logWarn
import psnc.m2dc.rtm.utils.printConsole
/*
#region description
......@@ -36,20 +35,21 @@ PCM algorithm:
#endregion
*/
class PCMAlgorithm(state: CurrentState,val algorithm: Solver.Algorithm) : RTMAlgorithm(state) {
private var solver: PCMSolver = PCMSolver(state.eval,algorithm)
class PCMAlgorithm(state: CurrentState, val algorithm: Solver.Algorithm) : RTMAlgorithm(state) {
private var solver: PCMSolver = PCMSolver(state.eval, algorithm)
init {
this.saveState()
if (this.unusedResourcesSuspendSavings() > 0) {
logWarn("there are some suspends savings:" + this.unusedResourcesSuspendSavings())
}
}
// var algoState: AlgorithmState? = null
private fun setSolver(powerBudget: Double) {
this.state.eval.addPowerBudget(powerBudget)
solver = PCMSolver(state.eval,this.algorithm)
solver = PCMSolver(state.eval, this.algorithm)
}
......@@ -57,7 +57,7 @@ class PCMAlgorithm(state: CurrentState,val algorithm: Solver.Algorithm) : RTMAlg
var suspendSavings = unusedResourcesSuspendSavings()
this.state.eval.resetPowerBudget(state.currentPowerOverLimit)
printConsole("PCM poweroverlimit: ${this.state.eval.powerOverLimit} $suspendSavings")
logInfo("PCM poweroverlimit: ${this.state.eval.powerOverLimit}, suspend: $suspendSavings")
if (suspendSavings > this.state.currentPowerOverLimit) {
logInfo("Suspend unused: Stop")
......@@ -67,7 +67,7 @@ class PCMAlgorithm(state: CurrentState,val algorithm: Solver.Algorithm) : RTMAlg
// var performanceViolation = checkPerformanceViolation(savings.toFloat())
if (this.solver.upperBoundCheck(null) > state.currentPowerOverLimit - suspendSavings) {
setSolver(suspendSavings)
printConsole("pcm poweroverlimit: ${this.state.eval.powerOverLimit}")
// printConsole("pcm poweroverlimit: ${this.state.eval.powerOverLimit}")
solver.solve()?.let {
it.forEach {
val state = this.state.nodeMap[it.key]!!.setPerformance(it.value)
......@@ -79,11 +79,11 @@ class PCMAlgorithm(state: CurrentState,val algorithm: Solver.Algorithm) : RTMAlg
return PowerSettings(result)
}
var savingsSum = 0.0
//val workLoadViolationList = ArrayList<Resource>()
//start intrusive actions
var nodeUtil = this.state.moveLastSuspendableUnused()
// setSolver()
var possibleSavings = this.solver.upperBoundCheck(null) > state.currentPowerOverLimit - suspendSavings
logInfo("Suspend")
while (!possibleSavings && nodeUtil != null) //&& it < 1000) {
{
val savings = getSuspendSavings(nodeUtil)
......@@ -102,7 +102,7 @@ class PCMAlgorithm(state: CurrentState,val algorithm: Solver.Algorithm) : RTMAlg
}
suspendSavings = unusedResourcesSuspendSavings()
setSolver(suspendSavings)
printConsole("pcm poweroverlimit: ${this.state.eval.powerOverLimit}")
// printConsole("pcm poweroverlimit: ${this.state.eval.powerOverLimit}")
val performanceSettings = solver.solve()
performanceSettings?.let {
it.forEach {
......
......@@ -6,7 +6,6 @@ import psnc.m2dc.rtm.ActionCache
import psnc.m2dc.rtm.CurrentState
import psnc.m2dc.rtm.StorageKeys
import psnc.m2dc.rtm.dao.DAO
import psnc.m2dc.rtm.dao.LightStorage
import psnc.m2dc.rtm.model.rest.PCMState
import psnc.m2dc.rtm.model.rest.PowerCappingMode
import psnc.m2dc.rtm.model.rest.PowerCappingStrategy
......@@ -67,9 +66,9 @@ class PCMApi() {
if (StorageKeys.POWER_LIMIT != null)
StorageKeys.POWER_LIMIT = null
if (Keys.PCM.power_limit_reset_all.getNullable() == true) {
ActionCache.resetPowerSettings(DAO.Resource.getNodes().keys)
ActionCache.restorePowerSettings(DAO.Resource.getNodes().keys)
} else
ActionCache.resetPowerSettings()
ActionCache.restorePowerSettings()
}
fun setPowerLimit(limit: Double): Boolean {
......@@ -83,7 +82,6 @@ class PCMApi() {
fun getPowerLimit(): Double? = StorageKeys.POWER_LIMIT ?: Keys.PCM.default_power_limit.get()
fun getCurrentState(): Any = PCMWorker.getCurrentState() ?: this.checkLimits()
......@@ -94,11 +92,9 @@ class PCMApi() {
fun checkState(): Boolean {
val limits = this.checkLimits()
// printConsole("checkLimits $limits")
if (limits.limitExceeded) {// || Keys.PCM.simple_performance_restore.get()) {
if (limits.limitExceeded) {
Timer.start()
val state = CurrentState(limits.currentPowerUsage)//+
// ((StorageKeys.POWER_LIMIT ?: 0.0) * Keys.PCM.power_buffer.get()))
val state = CurrentState(limits.currentPowerUsage)
Timer.stop()
PCMWorker.start(state)
......@@ -111,7 +107,7 @@ class PCMApi() {
fun checkLimits(): PCMState {
try {
Timer.start()
val currentPowerOverLimit: Pair<Double, Double> = Instance.getPowerOverLimit(true)
val currentPowerOverLimit: Pair<Double, Double> = Instance.getPowerOverLimit()
return PCMState(currentPowerOverLimit.first > 0, -currentPowerOverLimit.first,
currentPowerOverLimit.second)
} finally {
......@@ -121,7 +117,6 @@ class PCMApi() {
}
/**
* todo: move to other api
*/
......@@ -148,25 +143,20 @@ class PCMApi() {
/**
check if power limit is not exceeded
@return power over limit (negative power budget), current power usage
*/
fun getPowerOverLimit(getSparePower: Boolean = false): Pair<Double, Double> {
logDebug("Power limit:" + LightStorage.instance.getValue("PCM_POWER_LIMIT")?.type() + " : " + LightStorage.instance.getValue("PCM_POWER_LIMIT")?.get())
var totalLimit = StorageKeys.POWER_LIMIT
fun getPowerOverLimit(): Pair<Double, Double> {
val map: HashMap<String, Double> = if (this.getPowerCappingStrategy() == PowerCappingStrategy.LAZY) {
RPMApi.instance().getPowerUsageList()
} else {
this.getMaxPowerUsageList()
}
if (totalLimit == null) {
totalLimit = this.getMaxPowerUsageList().sum()
}
val totalUsage = map.sum()
// printConsole("check limit: ${totalUsage} limit: ${totalLimit}")
if (totalUsage >= totalLimit || getSparePower) {
return Pair(totalUsage - totalLimit, totalUsage)
}
return Pair(0.0, totalUsage)
logDebug("Power limit: ${StorageKeys.POWER_LIMIT}")
var totalLimit = StorageKeys.POWER_LIMIT ?: this.getMaxPowerUsageList().sum()
return Pair(totalUsage - totalLimit, totalUsage)
}
......
......@@ -73,12 +73,13 @@ class PCMWorker(var state: CurrentState) : RTMWorker() {
while (this.currentIteration < MAX_ITERATION && !stop) {
try {
if (this.checkLimits()) {
printConsole("""Current overLimit:${this.state.currentPowerOverLimit}""")
logInfo("""Current overLimit:${this.state.currentPowerOverLimit}""")
logInfo("Current power settings: " + ActionCache.toString())
val actions = algorithm.next()
notify(actions)
if (Keys.RTM.debug_actions.get()) {
printConsole(actions.getSettings().toJson())
this.algorithm.dumpSettings(actions)
} else {
printConsole(" New actions: " + actions.toJson())
startActions(actions, state)
......@@ -91,7 +92,7 @@ class PCMWorker(var state: CurrentState) : RTMWorker() {
}
printConsole("counter: $counter ${this.currentIteration}")
counter++
Thread.sleep(1000)
Thread.sleep(1500)
if ((counter > 15 /*minimum stabilization time*/
&& (this.currentIteration > MAX_ITERATION || this.currentIteration == 0))
|| counter > 60 /*maximum stabilization time*/) {
......
......@@ -9,6 +9,7 @@ import psnc.m2dc.rtm.rpm.RPMApi
import psnc.m2dc.rtm.utils.logError
import psnc.m2dc.rtm.utils.logInfo
import psnc.m2dc.rtm.utils.printConsole
import java.util.*
import java.util.concurrent.Semaphore
/**
......@@ -55,6 +56,8 @@ object RTMScannerWorker {
initLock!!.release()
try {
logInfo("RTM scan start")
var date = Date().time
val restore = {(Date().time - date) > 300000}
while (Keys.RTM.scan_thread.getBoolean() && !stop) {
try {
// logDebug("current scan timeout: ${Keys.RTM.scan_timeout_ms.getLong()}, counter: $iter ")
......@@ -72,6 +75,9 @@ object RTMScannerWorker {
}
}
RPMApi.instance().restartFailedActions()
if(restore()){
ActionCache.resetPowerSettings()
}
} catch (e: Exception) {
e.printStackTrace()
......
package psnc.m2dc.rtm.commands.java
val APP_VERSION = "1.0.632_49699"
\ No newline at end of file
val APP_VERSION = "1.0.634_54798"
\ No newline at end of file
package psnc.m2dc.rtm.client
const val APP_VERSION = "1.1.632_49701"
\ No newline at end of file
const val APP_VERSION = "1.1.634_54800"
\ No newline at end of file
......@@ -71,6 +71,9 @@ object ActionCache {
}
}
/**
* restart failed actions
*/
fun restartActions(restartCallback: (ResourceActionResult) -> Unit) {
synchronized(this.lock) {
if (!repeatActions.isEmpty()) {
......
......@@ -4,8 +4,6 @@ import psnc.m2dc.rtm.model.rpm.ComponentPowerModel
import psnc.m2dc.rtm.model.rpm.Resource
import psnc.m2dc.rtm.properties.keys.Keys
import psnc.m2dc.rtm.utils.logWarn
import psnc.m2dc.rtm.utils.printConsole
import psnc.m2dc.rtm.utils.toJson
/**
* @param utilization -cpuutilization or utilization of whole node -> (current_power-min_power)/(max_power-min_power)
......@@ -76,32 +74,40 @@ data class NodeUtil(val nodeId: String,
LoadProfile.DEFAULT.name, null,
timeInterval)
}
}
fun isInUse(_timeInterval: TimeInterval): Boolean {
@Suppress("SENSELESS_COMPARISON")
if (this.timeInterval == null) {
//HACK for gson parsing from json
this.timeInterval = TimeInterval.DEFAULT()
}
if (!this.timeInterval.contains(_timeInterval)) {
printConsole("Interval out of fate,${this.timeInterval.toJson()}")
return false
}
if (this.utilization ?: 100.0 > Keys.Default.min_active_load.get())/* && (diskUsage != null) */ {
printConsole("Active ${this.utilization}")
return true
}
this.subcomponentUtilization.values.forEach({
if (it > Keys.Default.min_active_component_load.get()) {
printConsole("Component is active: $it")
return true
}
})
return false
fun isActive(utilization: Double?) = utilization ?: 100.0 > Keys.Default.min_active_load.get()
// {
// if (utilization ?: 100.0 > Keys.Default.min_active_load.get())/* && (diskUsage != null) */ {
//// printConsole("Active ${this.utilization}")
// true
// }
// false
// }
}
// fun isInUse(_timeInterval: TimeInterval): Boolean {
// @Suppress("SENSELESS_COMPARISON")
// if (this.timeInterval == null) {
// //HACK for gson parsing from json
// this.timeInterval = TimeInterval.DEFAULT()
// }
// if (!this.timeInterval.contains(_timeInterval)) {
// printConsole("Interval out of date,${this.timeInterval.toJson()}")
// return false
// }
//
// if (this.utilization ?: 100.0 > Keys.Default.min_active_load.get())/* && (diskUsage != null) */ {
// printConsole("Active ${this.utilization}")
// return true
// }
// this.subcomponentUtilization.values.forEach({
// if (it > Keys.Default.min_active_component_load.get()) {
// printConsole("Component is active: $it")
// return true
// }
// })
// return false
// }
// fun getInactiveSubComponents(_timeInterval: TimeInterval): Map<String, String> {
......
......@@ -124,6 +124,13 @@ fun Any.toJson(path: String) {
f.writeText(RTMProperties.JSONParser.toJson(this))
}
fun String.store(path: String) {
val f = File(path)
File(f.parent).iF({ !it.exists() }, { it.mkdirs() })
val g = GsonBuilder().setPrettyPrinting().create()
f.writeText(this)
}
fun Any.toJson(path: String, pretty: Boolean = true) {
if (!pretty) {
return this.toJson(path)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment