Re: [PATCH v8 5/6] PM / devfreq: Add PM QoS support

[Leonard Crestez <leonard.crestez@nxp.com>]

On 2019-09-27 4:45 AM, Chanwoo Choi wrote:
> On 19. 9. 26. 오후 10:43, Leonard Crestez wrote:
>> On 2019-09-26 4:07 AM, Chanwoo Choi wrote:
>>> On 19. 9. 26. 오전 6:18, Leonard Crestez wrote:
>>>> On 25.09.2019 05:11, Chanwoo Choi wrote:
>>>>> On 19. 9. 24. 오후 7:11, Leonard Crestez wrote:
>>>>>> Register notifiers with the PM QoS framework in order to respond to
>>>>>> requests for DEV_PM_QOS_MIN_FREQUENCY and DEV_PM_QOS_MAX_FREQUENCY.
>>>>>>
>>>>>> No notifiers are added by this patch but PM QoS constraints can be
>>>>>> imposed externally (for example from other devices).
>>>>>>
>>>>>> Signed-off-by: Leonard Crestez <leonard.crestez@nxp.com>
>>>>>> Reviewed-by: Matthias Kaehlcke <mka@chromium.org>
>>>>>> ---
>>>>>>     drivers/devfreq/devfreq.c | 75 +++++++++++++++++++++++++++++++++++++++
>>>>>>     include/linux/devfreq.h   |  5 +++
>>>>>>     2 files changed, 80 insertions(+)
>>>>>>
>>>>>> diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
>>>>>> index eee403e70c84..784f3e40536a 100644
>>>>>> --- a/drivers/devfreq/devfreq.c
>>>>>> +++ b/drivers/devfreq/devfreq.c
>>>>>> @@ -22,15 +22,18 @@
>>>>>>     #include <linux/platform_device.h>
>>>>>>     #include <linux/list.h>
>>>>>>     #include <linux/printk.h>
>>>>>>     #include <linux/hrtimer.h>
>>>>>>     #include <linux/of.h>
>>>>>> +#include <linux/pm_qos.h>
>>>>>>     #include "governor.h"
>>>>>>     
>>>>>>     #define CREATE_TRACE_POINTS
>>>>>>     #include <trace/events/devfreq.h>
>>>>>>     
>>>>>> +#define HZ_PER_KHZ	1000
>>>>>> +
>>>>>>     static struct class *devfreq_class;
>>>>>>     
>>>>>>     /*
>>>>>>      * devfreq core provides delayed work based load monitoring helper
>>>>>>      * functions. Governors can use these or can implement their own
>>>>>> @@ -109,10 +112,11 @@ static unsigned long find_available_max_freq(struct devfreq *devfreq)
>>>>>>     static void get_freq_range(struct devfreq *devfreq,
>>>>>>     			   unsigned long *min_freq,
>>>>>>     			   unsigned long *max_freq)
>>>>>>     {
>>>>>>     	unsigned long *freq_table = devfreq->profile->freq_table;
>>>>>> +	unsigned long qos_min_freq, qos_max_freq;
>>>>>>     
>>>>>>     	lockdep_assert_held(&devfreq->lock);
>>>>>>     
>>>>>>     	/*
>>>>>>     	 * Init min/max frequency from freq table.
>>>>>> @@ -125,10 +129,18 @@ static void get_freq_range(struct devfreq *devfreq,
>>>>>>     	} else {
>>>>>>     		*min_freq = freq_table[devfreq->profile->max_state - 1];
>>>>>>     		*max_freq = freq_table[0];
>>>>>>     	}
>>>>>>     
>>>>>> +	/* constraints from PM QoS */
>>>>>
>>>>> As I commented on patch4,
>>>>> 'constraints' -> 'Constraint' because first verb have to be used
>>>>> as the sigular verbs.
>>>>
>>>> Already discussed for another patch; I will modify to "Apply constraints
>>>> from PM QoS" instead.
>>>
>>> I agree the new comment with 'Apply constraints ... '.
>>>
>>>>
>>>>> I prefer to use following comments:
>>>>>
>>>>> 	/* Constraint minimum/maximum frequency from PM QoS constraints */
>>>>>
>>>>>> +	qos_min_freq = dev_pm_qos_read_value(devfreq->dev.parent,
>>>>>> +					     DEV_PM_QOS_MIN_FREQUENCY);
>>>>>> +	qos_max_freq = dev_pm_qos_read_value(devfreq->dev.parent,
>>>>>> +					     DEV_PM_QOS_MIN_FREQUENCY);
>>>>>> +	*min_freq = max(*min_freq, HZ_PER_KHZ * qos_min_freq);
>>>>>> +	*max_freq = min(*max_freq, HZ_PER_KHZ * qos_max_freq);
>>>>>> +
>>>>>>     	/* constraints from sysfs */
>>>>>>     	*min_freq = max(*min_freq, devfreq->min_freq);
>>>>>>     	*max_freq = min(*max_freq, devfreq->max_freq);
>>>>>>     
>>>>>>     	/* constraints from OPP interface */
>>>>>> @@ -606,10 +618,49 @@ static int devfreq_notifier_call(struct notifier_block *nb, unsigned long type,
>>>>>>     	mutex_unlock(&devfreq->lock);
>>>>>>     
>>>>>>     	return ret;
>>>>>>     }
>>>>>>     
>>>>>> +/**
>>>>>> + * qos_notifier_call() - Common handler for QoS constraints.
>>>>>> + * @devfreq:    the devfreq instance.
>>>>>> + */
>>>>>> +static int qos_notifier_call(struct devfreq *devfreq)
>>>>>> +{
>>>>>> +	int err;
>>>>>> +
>>>>>> +	mutex_lock(&devfreq->lock);
>>>>>> +	err = update_devfreq(devfreq);
>>>>>> +	mutex_unlock(&devfreq->lock);
>>>>>> +	if (err)
>>>>>> +		dev_err(devfreq->dev.parent,
>>>>>> +				"failed to update frequency for PM QoS constraints (%d)\n",
>>>>>
>>>>> Is it not over 80 char?
>>>>
>>>> Yes but coding style explicitly forbids breaking strings.
>>>>
>>>>>> +				err);
>>>>>> +
>>>>>> +	return NOTIFY_OK;
>>>>>> +}
>>>>>> +
>>>>>> +/**
>>>>>> + * qos_min_notifier_call() - Callback for QoS min_freq changes.
>>>>>> + * @nb:		Should be devfreq->nb_min
>>>>>> + */
>>>>>> +static int qos_min_notifier_call(struct notifier_block *nb,
>>>>>> +					 unsigned long val, void *ptr)
>>>>>> +{
>>>>>> +	return qos_notifier_call(container_of(nb, struct devfreq, nb_min));
>>>>>> +}
>>>>>> +
>>>>>> +/**
>>>>>> + * qos_max_notifier_call() - Callback for QoS max_freq changes.
>>>>>> + * @nb:		Should be devfreq->nb_max
>>>>>> + */
>>>>>> +static int qos_max_notifier_call(struct notifier_block *nb,
>>>>>> +					 unsigned long val, void *ptr)
>>>>>> +{
>>>>>> +	return qos_notifier_call(container_of(nb, struct devfreq, nb_max));
>>>>>> +}
>>>>>> +
>>>>>>     /**
>>>>>>      * devfreq_dev_release() - Callback for struct device to release the device.
>>>>>>      * @dev:	the devfreq device
>>>>>>      *
>>>>>>      * Remove devfreq from the list and release its resources.
>>>>>> @@ -620,10 +671,15 @@ static void devfreq_dev_release(struct device *dev)
>>>>>>     
>>>>>>     	mutex_lock(&devfreq_list_lock);
>>>>>>     	list_del(&devfreq->node);
>>>>>>     	mutex_unlock(&devfreq_list_lock);
>>>>>>     
>>>>>> +	dev_pm_qos_remove_notifier(devfreq->dev.parent, &devfreq->nb_max,
>>>>>> +			DEV_PM_QOS_MAX_FREQUENCY);
>>>>>> +	dev_pm_qos_remove_notifier(devfreq->dev.parent, &devfreq->nb_min,
>>>>>> +			DEV_PM_QOS_MIN_FREQUENCY);
>>>>>> +
>>>>>
>>>>> Just print error with dev_err() without stopping the release step.
>>>>>
>>>>> I prefer to handle the return value if kernel API provides
>>>>> the error code.
>>>
>>> How about?
>>
>> Modified to dev_warn. This also applies to PATCH 6 so I replied there.
>>
>>>>>>     	if (devfreq->profile->exit)
>>>>>>     		devfreq->profile->exit(devfreq->dev.parent);
>>>>>>     
>>>>>>     	kfree(devfreq->time_in_state);
>>>>>>     	kfree(devfreq->trans_table);
>>>>>> @@ -733,10 +789,28 @@ struct devfreq *devfreq_add_device(struct device *dev,
>>>>>>     	if (err) {
>>>>>>     		put_device(&devfreq->dev);
>>>>>>     		goto err_out;
>>>>>>     	}
>>>>>>     
>>>>>> +	/*
>>>>>> +	 * Register notifiers for updates to min/max_freq after device is
>>>>>> +	 * initialized (and we can handle notifications) but before the
>>>>>> +	 * governor is started (which should do an initial enforcement of
>>>>>> +	 * constraints).
>>>>>> +	 */
>>>>>
>>>>> My previous comment is not enough why I prefer to remove it. Sorry.
>>>>> Actually, until now, the devfreq_add_device() don't have the detailed
>>>>> comments because the line code is not too long. But, at the present time,
>>>>> devfreq_add_device() is too long. It means that the detailed comment
>>>>> are necessary.
>>>>>
>>>>> So, I'll add the detailed comment for each step of devfreq_add_device()
>>>>> on separate patch to keep the same style. I'll send the patch to you
>>>>> for the review.
>>>>
>>>> This is very likely to result in merge conflicts, maybe wait for my
>>>> series to go in first?
>>>
>>> I'll send the separate patch after finished the review of these patches.
>>> So, if you agree, please remove this comment on this patch.
>>>
>>> You can review the detailed comments on separate patch when I send.
>> This patch already contains comments and they explain the code being
>> added. Doesn't it make more sense for comments and code to go in together?
>>
>> I think the comment is a resonable explanation as to why notifiers are
>> registered at that specific step in the initialization sequence.
> 
> If you add this comment on this patch, OK. just I have some comments.
> 
> 	/*
> 	 * Register notifiers for updates to min/max_freq after device is
> 	 * initialized (and we can handle notifications) but before the
> 
> I think that 'device is initialized' is not needed.
> It is always true, it don't need to add the additional comments.
> because dev_pm_qos_add_notifier() must need the 'devfreq->dev'.
> The this code prove the call sequence between them.

In theory if a notifier is registered too soon then it could crash on a 
NULL pointer. But looking at the code it first checks that "governor != 
NULL) so it would be harmless.

> 
> About 'us', don't use 'we'. The subject is 'devfreq' or other device instead of us.
> 
> 	 * governor is started (which should do an initial enforcement of
> 	 * constraints).
>   	 */
> 
> Actually, it doesn't matter the initialization step between governor
> and PM_QOS registration.

In theory PM QoS constraints could be modified between governor startup 
and notifier registration and that update would be lost (until the next 
one).

> If start governor and then register PM_QOS,
> the governor can decide the new frequency. And then PM_QOS request
> the new constraints for min/max frequency. On request time of PM_QOS,
> devfreq can detect this time and then apply the constraints from PM_QOS.
> The user of devfreq's PM_QOS must need the phandle or device instance
> of devfreq deivce. It means that user of devfreq's PM_QOS can request
> the any constraints of PM_QOS_MIN/MAX through PM_QOS interface.
> 
> So, it doesn't need to add the following comments
> because following comment is not always mandatory.
> " governor is started (which should do an initial enforcement of constraints)."
> 
> Also, you don't need to use parentheses on comments.
> 
> Actually, I think that following comments are enough.
> 
> 	/* Register PM QoS notifiers for updating to min/max freq */

I thought that explaning why it's done at this particular step but I'll 
just remove it, devfreq_add_device can get more comments later.

>>>>>> +	devfreq->nb_min.notifier_call = qos_min_notifier_call;
>>>>>> +	err = dev_pm_qos_add_notifier(devfreq->dev.parent, &devfreq->nb_min,
>>>>>> +				      DEV_PM_QOS_MIN_FREQUENCY);
>>>>>> +	if (err)
>>>>>> +		goto err_devfreq;
>>>>>> +
>>>>>> +	devfreq->nb_max.notifier_call = qos_max_notifier_call;
>>>>>> +	err = dev_pm_qos_add_notifier(devfreq->dev.parent, &devfreq->nb_max,
>>>>>> +				      DEV_PM_QOS_MAX_FREQUENCY);
>>>>>> +	if (err)
>>>>>> +		goto err_devfreq;
>>>>>> +
>>>>>>     	mutex_lock(&devfreq_list_lock);
>>>>>>     
>>>>>>     	governor = try_then_request_governor(devfreq->governor_name);
>>>>>>     	if (IS_ERR(governor)) {
>>>>>>     		dev_err(dev, "%s: Unable to find governor for the device\n",
>>>>>> @@ -760,10 +834,11 @@ struct devfreq *devfreq_add_device(struct device *dev,
>>>>>>     
>>>>>>     	return devfreq;
>>>>>>     
>>>>>>     err_init:
>>>>>>     	mutex_unlock(&devfreq_list_lock);
>>>>>> +err_devfreq:
>>>>>>     	devfreq_remove_device(devfreq);
>>>>>>     	return ERR_PTR(err);
>>>>>>     
>>>>>>     err_dev:
>>>>>>     	/*
>>>>>> diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h
>>>>>> index c3cbc15fdf08..dac0dffeabb4 100644
>>>>>> --- a/include/linux/devfreq.h
>>>>>> +++ b/include/linux/devfreq.h
>>>>>> @@ -134,10 +134,12 @@ struct devfreq_dev_profile {
>>>>>>      * @total_trans:	Number of devfreq transitions
>>>>>>      * @trans_table:	Statistics of devfreq transitions
>>>>>>      * @time_in_state:	Statistics of devfreq states
>>>>>>      * @last_stat_updated:	The last time stat updated
>>>>>>      * @transition_notifier_list: list head of DEVFREQ_TRANSITION_NOTIFIER notifier
>>>>>> + * @nb_min:		Notifier block for DEV_PM_QOS_MIN_FREQUENCY
>>>>>> + * @nb_max:		Notifier block for DEV_PM_QOS_MAX_FREQUENCY
>>>>>>      *
>>>>>>      * This structure stores the devfreq information for a give device.
>>>>>>      *
>>>>>>      * Note that when a governor accesses entries in struct devfreq in its
>>>>>>      * functions except for the context of callbacks defined in struct
>>>>>> @@ -176,10 +178,13 @@ struct devfreq {
>>>>>>     	unsigned int *trans_table;
>>>>>>     	unsigned long *time_in_state;
>>>>>>     	unsigned long last_stat_updated;
>>>>>>     
>>>>>>     	struct srcu_notifier_head transition_notifier_list;
>>>>>> +
>>>>>> +	struct notifier_block nb_min;
>>>>>> +	struct notifier_block nb_max;
>>>>>>     };
>>>>>>     
>>>>>>     struct devfreq_freqs {
>>>>>>     	unsigned long old;
>>>>>>     	unsigned long new;
> 
>