1// SPDX-License-Identifier: GPL-2.0-only
2//
3// Copyright (C) 2020-2021 Samuel Holland <samuel@sholland.org>
4//
5
6#include <linux/clk.h>
7#include <linux/devfreq.h>
8#include <linux/err.h>
9#include <linux/io.h>
10#include <linux/iopoll.h>
11#include <linux/module.h>
12#include <linux/of.h>
13#include <linux/platform_device.h>
14#include <linux/property.h>
15
16#define MBUS_CR				0x0000
17#define MBUS_CR_GET_DRAM_TYPE(x)	(((x) >> 16) & 0x7)
18#define MBUS_CR_DRAM_TYPE_DDR2		2
19#define MBUS_CR_DRAM_TYPE_DDR3		3
20#define MBUS_CR_DRAM_TYPE_DDR4		4
21#define MBUS_CR_DRAM_TYPE_LPDDR2	6
22#define MBUS_CR_DRAM_TYPE_LPDDR3	7
23
24#define MBUS_TMR			0x000c
25#define MBUS_TMR_PERIOD(x)		((x) - 1)
26
27#define MBUS_PMU_CFG			0x009c
28#define MBUS_PMU_CFG_PERIOD(x)		(((x) - 1) << 16)
29#define MBUS_PMU_CFG_UNIT		(0x3 << 1)
30#define MBUS_PMU_CFG_UNIT_B		(0x0 << 1)
31#define MBUS_PMU_CFG_UNIT_KB		(0x1 << 1)
32#define MBUS_PMU_CFG_UNIT_MB		(0x2 << 1)
33#define MBUS_PMU_CFG_ENABLE		(0x1 << 0)
34
35#define MBUS_PMU_BWCR(n)		(0x00a0 + (0x04 * (n)))
36
37#define MBUS_TOTAL_BWCR			MBUS_PMU_BWCR(5)
38#define MBUS_TOTAL_BWCR_H616		MBUS_PMU_BWCR(13)
39
40#define MBUS_MDFSCR			0x0100
41#define MBUS_MDFSCR_BUFFER_TIMING	(0x1 << 15)
42#define MBUS_MDFSCR_PAD_HOLD		(0x1 << 13)
43#define MBUS_MDFSCR_BYPASS		(0x1 << 4)
44#define MBUS_MDFSCR_MODE		(0x1 << 1)
45#define MBUS_MDFSCR_MODE_DFS		(0x0 << 1)
46#define MBUS_MDFSCR_MODE_CFS		(0x1 << 1)
47#define MBUS_MDFSCR_START		(0x1 << 0)
48
49#define MBUS_MDFSMRMR			0x0108
50
51#define DRAM_PWRCTL			0x0004
52#define DRAM_PWRCTL_SELFREF_EN		(0x1 << 0)
53
54#define DRAM_RFSHTMG			0x0090
55#define DRAM_RFSHTMG_TREFI(x)		((x) << 16)
56#define DRAM_RFSHTMG_TRFC(x)		((x) << 0)
57
58#define DRAM_VTFCR			0x00b8
59#define DRAM_VTFCR_VTF_ENABLE		(0x3 << 8)
60
61#define DRAM_ODTMAP			0x0120
62
63#define DRAM_DX_MAX			4
64
65#define DRAM_DXnGCR0(n)			(0x0344 + 0x80 * (n))
66#define DRAM_DXnGCR0_DXODT		(0x3 << 4)
67#define DRAM_DXnGCR0_DXODT_DYNAMIC	(0x0 << 4)
68#define DRAM_DXnGCR0_DXODT_ENABLED	(0x1 << 4)
69#define DRAM_DXnGCR0_DXODT_DISABLED	(0x2 << 4)
70#define DRAM_DXnGCR0_DXEN		(0x1 << 0)
71
72struct sun8i_a33_mbus_variant {
73	u32					min_dram_divider;
74	u32					max_dram_divider;
75	u32					odt_freq_mhz;
76};
77
78struct sun8i_a33_mbus {
79	const struct sun8i_a33_mbus_variant	*variant;
80	void __iomem				*reg_dram;
81	void __iomem				*reg_mbus;
82	struct clk				*clk_bus;
83	struct clk				*clk_dram;
84	struct clk				*clk_mbus;
85	struct devfreq				*devfreq_dram;
86	struct devfreq_simple_ondemand_data	gov_data;
87	struct devfreq_dev_profile		profile;
88	u32					data_width;
89	u32					nominal_bw;
90	u32					odtmap;
91	u32					tREFI_ns;
92	u32					tRFC_ns;
93	unsigned long				freq_table[];
94};
95
96/*
97 * The unit for this value is (MBUS clock cycles / MBUS_TMR_PERIOD). When
98 * MBUS_TMR_PERIOD is programmed to match the MBUS clock frequency in MHz, as
99 * it is during DRAM init and during probe, the resulting unit is microseconds.
100 */
101static int pmu_period = 50000;
102module_param(pmu_period, int, 0644);
103MODULE_PARM_DESC(pmu_period, "Bandwidth measurement period (microseconds)");
104
105static u32 sun8i_a33_mbus_get_peak_bw(struct sun8i_a33_mbus *priv)
106{
107	/* Returns the peak transfer (in KiB) during any single PMU period. */
108	return readl_relaxed(priv->reg_mbus + MBUS_TOTAL_BWCR);
109}
110
111static void sun8i_a33_mbus_restart_pmu_counters(struct sun8i_a33_mbus *priv)
112{
113	u32 pmu_cfg = MBUS_PMU_CFG_PERIOD(pmu_period) | MBUS_PMU_CFG_UNIT_KB;
114
115	/* All PMU counters are cleared on a disable->enable transition. */
116	writel_relaxed(pmu_cfg,
117		       priv->reg_mbus + MBUS_PMU_CFG);
118	writel_relaxed(pmu_cfg | MBUS_PMU_CFG_ENABLE,
119		       priv->reg_mbus + MBUS_PMU_CFG);
120
121}
122
123static void sun8i_a33_mbus_update_nominal_bw(struct sun8i_a33_mbus *priv,
124					     u32 ddr_freq_mhz)
125{
126	/*
127	 * Nominal bandwidth (KiB per PMU period):
128	 *
129	 *   DDR transfers   microseconds     KiB
130	 *   ------------- * ------------ * --------
131	 *    microsecond     PMU period    transfer
132	 */
133	priv->nominal_bw = ddr_freq_mhz * pmu_period * priv->data_width / 1024;
134}
135
136static int sun8i_a33_mbus_set_dram_freq(struct sun8i_a33_mbus *priv,
137					unsigned long freq)
138{
139	u32  ddr_freq_mhz = freq / USEC_PER_SEC; /* DDR */
140	u32 dram_freq_mhz =    ddr_freq_mhz / 2; /* SDR */
141	u32 mctl_freq_mhz =   dram_freq_mhz / 2; /* HDR */
142	u32 dxodt, mdfscr, pwrctl, vtfcr;
143	u32 i, tREFI_32ck, tRFC_ck;
144	int ret;
145
146	/* The rate change is not effective until the MDFS process runs. */
147	ret = clk_set_rate(priv->clk_dram, freq);
148	if (ret)
149		return ret;
150
151	/* Disable automatic self-refesh and VTF before starting MDFS. */
152	pwrctl = readl_relaxed(priv->reg_dram + DRAM_PWRCTL) &
153		 ~DRAM_PWRCTL_SELFREF_EN;
154	writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
155	vtfcr = readl_relaxed(priv->reg_dram + DRAM_VTFCR);
156	writel_relaxed(vtfcr & ~DRAM_VTFCR_VTF_ENABLE,
157		       priv->reg_dram + DRAM_VTFCR);
158
159	/* Set up MDFS and enable double buffering for timing registers. */
160	mdfscr = MBUS_MDFSCR_MODE_DFS |
161		 MBUS_MDFSCR_BYPASS |
162		 MBUS_MDFSCR_PAD_HOLD |
163		 MBUS_MDFSCR_BUFFER_TIMING;
164	writel(mdfscr, priv->reg_mbus + MBUS_MDFSCR);
165
166	/* Update the buffered copy of RFSHTMG. */
167	tREFI_32ck = priv->tREFI_ns * mctl_freq_mhz / 1000 / 32;
168	tRFC_ck = DIV_ROUND_UP(priv->tRFC_ns * mctl_freq_mhz, 1000);
169	writel(DRAM_RFSHTMG_TREFI(tREFI_32ck) | DRAM_RFSHTMG_TRFC(tRFC_ck),
170	       priv->reg_dram + DRAM_RFSHTMG);
171
172	/* Enable ODT if needed, or disable it to save power. */
173	if (priv->odtmap && dram_freq_mhz > priv->variant->odt_freq_mhz) {
174		dxodt = DRAM_DXnGCR0_DXODT_DYNAMIC;
175		writel(priv->odtmap, priv->reg_dram + DRAM_ODTMAP);
176	} else {
177		dxodt = DRAM_DXnGCR0_DXODT_DISABLED;
178		writel(0, priv->reg_dram + DRAM_ODTMAP);
179	}
180	for (i = 0; i < DRAM_DX_MAX; ++i) {
181		void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
182
183		writel((readl(reg) & ~DRAM_DXnGCR0_DXODT) | dxodt, reg);
184	}
185
186	dev_dbg(priv->devfreq_dram->dev.parent,
187		"Setting DRAM to %u MHz, tREFI=%u, tRFC=%u, ODT=%s\n",
188		dram_freq_mhz, tREFI_32ck, tRFC_ck,
189		dxodt == DRAM_DXnGCR0_DXODT_DYNAMIC ? "dynamic" : "disabled");
190
191	/* Trigger hardware MDFS. */
192	writel(mdfscr | MBUS_MDFSCR_START, priv->reg_mbus + MBUS_MDFSCR);
193	ret = readl_poll_timeout_atomic(priv->reg_mbus + MBUS_MDFSCR, mdfscr,
194					!(mdfscr & MBUS_MDFSCR_START), 10, 1000);
195	if (ret)
196		return ret;
197
198	/* Disable double buffering. */
199	writel(0, priv->reg_mbus + MBUS_MDFSCR);
200
201	/* Restore VTF configuration. */
202	writel_relaxed(vtfcr, priv->reg_dram + DRAM_VTFCR);
203
204	/* Enable automatic self-refresh at the lowest frequency only. */
205	if (freq == priv->freq_table[0])
206		pwrctl |= DRAM_PWRCTL_SELFREF_EN;
207	writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
208
209	sun8i_a33_mbus_restart_pmu_counters(priv);
210	sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq_mhz);
211
212	return 0;
213}
214
215static int sun8i_a33_mbus_set_dram_target(struct device *dev,
216					  unsigned long *freq, u32 flags)
217{
218	struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
219	struct devfreq *devfreq = priv->devfreq_dram;
220	struct dev_pm_opp *opp;
221	int ret;
222
223	opp = devfreq_recommended_opp(dev, freq, flags);
224	if (IS_ERR(opp))
225		return PTR_ERR(opp);
226
227	dev_pm_opp_put(opp);
228
229	if (*freq == devfreq->previous_freq)
230		return 0;
231
232	ret = sun8i_a33_mbus_set_dram_freq(priv, *freq);
233	if (ret) {
234		dev_warn(dev, "failed to set DRAM frequency: %d\n", ret);
235		*freq = devfreq->previous_freq;
236	}
237
238	return ret;
239}
240
241static int sun8i_a33_mbus_get_dram_status(struct device *dev,
242					  struct devfreq_dev_status *stat)
243{
244	struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
245
246	stat->busy_time		= sun8i_a33_mbus_get_peak_bw(priv);
247	stat->total_time	= priv->nominal_bw;
248	stat->current_frequency	= priv->devfreq_dram->previous_freq;
249
250	sun8i_a33_mbus_restart_pmu_counters(priv);
251
252	dev_dbg(dev, "Using %lu/%lu (%lu%%) at %lu MHz\n",
253		stat->busy_time, stat->total_time,
254		DIV_ROUND_CLOSEST(stat->busy_time * 100, stat->total_time),
255		stat->current_frequency / USEC_PER_SEC);
256
257	return 0;
258}
259
260static int sun8i_a33_mbus_hw_init(struct device *dev,
261				  struct sun8i_a33_mbus *priv,
262				  unsigned long ddr_freq)
263{
264	u32 i, mbus_cr, mbus_freq_mhz;
265
266	/* Choose tREFI and tRFC to match the configured DRAM type. */
267	mbus_cr = readl_relaxed(priv->reg_mbus + MBUS_CR);
268	switch (MBUS_CR_GET_DRAM_TYPE(mbus_cr)) {
269	case MBUS_CR_DRAM_TYPE_DDR2:
270	case MBUS_CR_DRAM_TYPE_DDR3:
271	case MBUS_CR_DRAM_TYPE_DDR4:
272		priv->tREFI_ns = 7800;
273		priv->tRFC_ns = 350;
274		break;
275	case MBUS_CR_DRAM_TYPE_LPDDR2:
276	case MBUS_CR_DRAM_TYPE_LPDDR3:
277		priv->tREFI_ns = 3900;
278		priv->tRFC_ns = 210;
279		break;
280	default:
281		return -EINVAL;
282	}
283
284	/* Save ODTMAP so it can be restored when raising the frequency. */
285	priv->odtmap = readl_relaxed(priv->reg_dram + DRAM_ODTMAP);
286
287	/* Compute the DRAM data bus width by counting enabled DATx8 blocks. */
288	for (i = 0; i < DRAM_DX_MAX; ++i) {
289		void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
290
291		if (!(readl_relaxed(reg) & DRAM_DXnGCR0_DXEN))
292			break;
293	}
294	priv->data_width = i;
295
296	dev_dbg(dev, "Detected %u-bit %sDDRx with%s ODT\n",
297		priv->data_width * 8,
298		MBUS_CR_GET_DRAM_TYPE(mbus_cr) > 4 ? "LP" : "",
299		priv->odtmap ? "" : "out");
300
301	/* Program MBUS_TMR such that the PMU period unit is microseconds. */
302	mbus_freq_mhz = clk_get_rate(priv->clk_mbus) / USEC_PER_SEC;
303	writel_relaxed(MBUS_TMR_PERIOD(mbus_freq_mhz),
304		       priv->reg_mbus + MBUS_TMR);
305
306	/* "Master Ready Mask Register" bits must be set or MDFS will block. */
307	writel_relaxed(0xffffffff, priv->reg_mbus + MBUS_MDFSMRMR);
308
309	sun8i_a33_mbus_restart_pmu_counters(priv);
310	sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq / USEC_PER_SEC);
311
312	return 0;
313}
314
315static int __maybe_unused sun8i_a33_mbus_suspend(struct device *dev)
316{
317	struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
318
319	clk_disable_unprepare(priv->clk_bus);
320
321	return 0;
322}
323
324static int __maybe_unused sun8i_a33_mbus_resume(struct device *dev)
325{
326	struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
327
328	return clk_prepare_enable(priv->clk_bus);
329}
330
331static int sun8i_a33_mbus_probe(struct platform_device *pdev)
332{
333	const struct sun8i_a33_mbus_variant *variant;
334	struct device *dev = &pdev->dev;
335	struct sun8i_a33_mbus *priv;
336	unsigned long base_freq;
337	unsigned int max_state;
338	const char *err;
339	int i, ret;
340
341	variant = device_get_match_data(dev);
342	if (!variant)
343		return -EINVAL;
344
345	max_state = variant->max_dram_divider - variant->min_dram_divider + 1;
346
347	priv = devm_kzalloc(dev, struct_size(priv, freq_table, max_state), GFP_KERNEL);
348	if (!priv)
349		return -ENOMEM;
350
351	platform_set_drvdata(pdev, priv);
352
353	priv->variant = variant;
354
355	priv->reg_dram = devm_platform_ioremap_resource_byname(pdev, "dram");
356	if (IS_ERR(priv->reg_dram))
357		return PTR_ERR(priv->reg_dram);
358
359	priv->reg_mbus = devm_platform_ioremap_resource_byname(pdev, "mbus");
360	if (IS_ERR(priv->reg_mbus))
361		return PTR_ERR(priv->reg_mbus);
362
363	priv->clk_bus = devm_clk_get(dev, "bus");
364	if (IS_ERR(priv->clk_bus))
365		return dev_err_probe(dev, PTR_ERR(priv->clk_bus),
366				     "failed to get bus clock\n");
367
368	priv->clk_dram = devm_clk_get(dev, "dram");
369	if (IS_ERR(priv->clk_dram))
370		return dev_err_probe(dev, PTR_ERR(priv->clk_dram),
371				     "failed to get dram clock\n");
372
373	priv->clk_mbus = devm_clk_get(dev, "mbus");
374	if (IS_ERR(priv->clk_mbus))
375		return dev_err_probe(dev, PTR_ERR(priv->clk_mbus),
376				     "failed to get mbus clock\n");
377
378	ret = clk_prepare_enable(priv->clk_bus);
379	if (ret)
380		return dev_err_probe(dev, ret,
381				     "failed to enable bus clock\n");
382
383	/* Lock the DRAM clock rate to keep priv->nominal_bw in sync. */
384	ret = clk_rate_exclusive_get(priv->clk_dram);
385	if (ret) {
386		err = "failed to lock dram clock rate\n";
387		goto err_disable_bus;
388	}
389
390	/* Lock the MBUS clock rate to keep MBUS_TMR_PERIOD in sync. */
391	ret = clk_rate_exclusive_get(priv->clk_mbus);
392	if (ret) {
393		err = "failed to lock mbus clock rate\n";
394		goto err_unlock_dram;
395	}
396
397	priv->gov_data.upthreshold	= 10;
398	priv->gov_data.downdifferential	=  5;
399
400	priv->profile.initial_freq	= clk_get_rate(priv->clk_dram);
401	priv->profile.polling_ms	= 1000;
402	priv->profile.target		= sun8i_a33_mbus_set_dram_target;
403	priv->profile.get_dev_status	= sun8i_a33_mbus_get_dram_status;
404	priv->profile.freq_table	= priv->freq_table;
405	priv->profile.max_state		= max_state;
406
407	ret = devm_pm_opp_set_clkname(dev, "dram");
408	if (ret) {
409		err = "failed to add OPP table\n";
410		goto err_unlock_mbus;
411	}
412
413	base_freq = clk_get_rate(clk_get_parent(priv->clk_dram));
414	for (i = 0; i < max_state; ++i) {
415		unsigned int div = variant->max_dram_divider - i;
416
417		priv->freq_table[i] = base_freq / div;
418
419		ret = dev_pm_opp_add(dev, priv->freq_table[i], 0);
420		if (ret) {
421			err = "failed to add OPPs\n";
422			goto err_remove_opps;
423		}
424	}
425
426	ret = sun8i_a33_mbus_hw_init(dev, priv, priv->profile.initial_freq);
427	if (ret) {
428		err = "failed to init hardware\n";
429		goto err_remove_opps;
430	}
431
432	priv->devfreq_dram = devfreq_add_device(dev, &priv->profile,
433						DEVFREQ_GOV_SIMPLE_ONDEMAND,
434						&priv->gov_data);
435	if (IS_ERR(priv->devfreq_dram)) {
436		ret = PTR_ERR(priv->devfreq_dram);
437		err = "failed to add devfreq device\n";
438		goto err_remove_opps;
439	}
440
441	/*
442	 * This must be set manually after registering the devfreq device,
443	 * because there is no way to select a dynamic OPP as the suspend OPP.
444	 */
445	priv->devfreq_dram->suspend_freq = priv->freq_table[0];
446
447	return 0;
448
449err_remove_opps:
450	dev_pm_opp_remove_all_dynamic(dev);
451err_unlock_mbus:
452	clk_rate_exclusive_put(priv->clk_mbus);
453err_unlock_dram:
454	clk_rate_exclusive_put(priv->clk_dram);
455err_disable_bus:
456	clk_disable_unprepare(priv->clk_bus);
457
458	return dev_err_probe(dev, ret, err);
459}
460
461static int sun8i_a33_mbus_remove(struct platform_device *pdev)
462{
463	struct sun8i_a33_mbus *priv = platform_get_drvdata(pdev);
464	unsigned long initial_freq = priv->profile.initial_freq;
465	struct device *dev = &pdev->dev;
466	int ret;
467
468	devfreq_remove_device(priv->devfreq_dram);
469
470	ret = sun8i_a33_mbus_set_dram_freq(priv, initial_freq);
471	if (ret)
472		dev_warn(dev, "failed to restore DRAM frequency: %d\n", ret);
473
474	dev_pm_opp_remove_all_dynamic(dev);
475	clk_rate_exclusive_put(priv->clk_mbus);
476	clk_rate_exclusive_put(priv->clk_dram);
477	clk_disable_unprepare(priv->clk_bus);
478
479	return 0;
480}
481
482static const struct sun8i_a33_mbus_variant sun50i_a64_mbus = {
483	.min_dram_divider	= 1,
484	.max_dram_divider	= 4,
485	.odt_freq_mhz		= 400,
486};
487
488static const struct of_device_id sun8i_a33_mbus_of_match[] = {
489	{ .compatible = "allwinner,sun50i-a64-mbus", .data = &sun50i_a64_mbus },
490	{ .compatible = "allwinner,sun50i-h5-mbus", .data = &sun50i_a64_mbus },
491	{ },
492};
493MODULE_DEVICE_TABLE(of, sun8i_a33_mbus_of_match);
494
495static SIMPLE_DEV_PM_OPS(sun8i_a33_mbus_pm_ops,
496			 sun8i_a33_mbus_suspend, sun8i_a33_mbus_resume);
497
498static struct platform_driver sun8i_a33_mbus_driver = {
499	.probe	= sun8i_a33_mbus_probe,
500	.remove	= sun8i_a33_mbus_remove,
501	.driver	= {
502		.name		= "sun8i-a33-mbus",
503		.of_match_table	= sun8i_a33_mbus_of_match,
504		.pm		= pm_ptr(&sun8i_a33_mbus_pm_ops),
505	},
506};
507module_platform_driver(sun8i_a33_mbus_driver);
508
509MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");
510MODULE_DESCRIPTION("Allwinner sun8i/sun50i MBUS DEVFREQ Driver");
511MODULE_LICENSE("GPL v2");
512